From d8ac63c94298873651cd59fad69ff8b55df432c1 Mon Sep 17 00:00:00 2001 From: James Vaughan Date: Sat, 8 Aug 2026 16:03:03 -0400 Subject: [PATCH] Update CSVReader to work with Linux and Windows Line endings In the current implementation you would get an extra line on Linux, or all of the lines would group together if using a Mac saved CSV file. --- src/TMG-Framework/Utilities/CSVReader.cs | 22 +++- .../Utilities/TestCSVReader.cs | 100 +++++++++++++++--- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/src/TMG-Framework/Utilities/CSVReader.cs b/src/TMG-Framework/Utilities/CSVReader.cs index d161659..c89eaaa 100644 --- a/src/TMG-Framework/Utilities/CSVReader.cs +++ b/src/TMG-Framework/Utilities/CSVReader.cs @@ -235,12 +235,13 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); } columns = numberOfColumns; - return true; + return columns > 0; } } c = DataBuffer[DataBufferPosition++]; - if ((c == '\n') || (c == '\0')) + if ((prevC == '\r' && c != '\n' && c != '\0') || (c == '\n') || (c == '\0')) { + // Create a new column if this is a Linux file format. if (prevC != '\r') { if (Data.Length <= numberOfColumns) @@ -250,6 +251,12 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); addOne = false; } + else if (prevC == '\r' && c != '\n' && c != '\0') + { + // If it was just a MAC encoded line ending then we need to process the current character in the next load. + DataBufferPosition--; + addOne = false; + } break; } if (c == '"') @@ -322,12 +329,13 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); } columns = numberOfColumns; - return true; + return columns > 0; } } c = DataBuffer[DataBufferPosition++]; - if ((c == '\n') || (c == '\0')) + if ((prevC == '\r' && c != '\n' && c != '\0') || (c == '\n') || (c == '\0')) { + // Create a new column if this is a Linux file format. if (prevC != '\r') { if (Data.Length <= numberOfColumns) @@ -337,6 +345,12 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); addOne = false; } + else if (prevC == '\r' && c != '\n' && c != '\0') + { + // If it was just a MAC encoded line ending then we need to process the current character in the next load. + DataBufferPosition--; + addOne = false; + } break; } if (c == '"') diff --git a/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs b/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs index 1336e3d..6215c20 100644 --- a/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs +++ b/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs @@ -31,46 +31,67 @@ namespace XTMF.Testing.TMG.Data [TestClass] public class TestCSVReader { - private readonly string[] TestCSVFileNames = new[] { "CSVTest1.csv", "CSVTest2.csv", "CSVTest3.csv", "CSVTest4.csv", "CSVTest5.csv", "CSVTest6.csv" }; + private readonly string[] TestCSVFileNames = new[] { "CSVTest1.csv", "CSVTest2.csv", "CSVTest3.csv", "CSVTest4.csv", "CSVTest5.csv", "CSVTest6.csv", "CSVTest7.csv", "CSVTest8.csv" }; [TestInitialize] public void CreateTestEnvironment() { if (!IsEnvironmentLoaded()) { - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[0])) + using (StreamWriter writer = new(TestCSVFileNames[0])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("1,2,3,4,5"); writer.WriteLine("3,1,4,5,2"); writer.WriteLine("1.23,4.56,7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[1])) + using (StreamWriter writer = new(TestCSVFileNames[1])) { writer.WriteLine("\"A\",\"B\",\"C\",\"D\",\"E\""); writer.WriteLine("\"1\",\"2\",3,\"4\",5"); writer.WriteLine("3,1,\"4\",5,2"); writer.WriteLine("1.23,\"4.56\",7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[2])) + using (StreamWriter writer = new(TestCSVFileNames[2])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("1,2,3,4,5"); writer.WriteLine("3,1,4,5,2"); writer.WriteLine("1.23,4.56,7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[3])) + using (StreamWriter writer = new(TestCSVFileNames[3])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("1,2,3,4,5"); writer.WriteLine("3,1,4,5,2"); writer.Write("1.23,4.56,7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[4])) + using (StreamWriter writer = new(TestCSVFileNames[4])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("\"abc\"\"1\",2,3,4,5"); } + + // The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Windows format. + using (StreamWriter writer = new(TestCSVFileNames[5])) + { + writer.Write("A,B,C,D,E\r\n"); + writer.Write("\"abc\"\"1\",2,3,4,5\r\n"); + } + + // The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Linux format. + using (StreamWriter writer = new(TestCSVFileNames[6])) + { + writer.Write("A,B,C,D,E\n"); + writer.Write("\"abc\"\"1\",2,3,4,5\n"); + } + + // The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Mac format. + using (StreamWriter writer = new(TestCSVFileNames[7])) + { + writer.Write("A,B,C,D,E\r"); + writer.Write("\"abc\"\"1\",2,3,4,5\r"); + } } } @@ -86,7 +107,7 @@ public bool IsEnvironmentLoaded() [TestMethod] public void TestCSVHeaders() { - using CsvReader reader = new CsvReader(TestCSVFileNames[0]); + using CsvReader reader = new(TestCSVFileNames[0]); //"A,B,C,D,E" var headers = reader.Headers; for (int i = 0; i < 5; i++) @@ -98,7 +119,7 @@ public void TestCSVHeaders() [TestMethod] public void TestQuotes() { - using CsvReader reader = new CsvReader(TestCSVFileNames[1]); + using CsvReader reader = new(TestCSVFileNames[1]); //"A,B,C,D,E" var headers = reader.Headers; for (int i = 0; i < 5; i++) @@ -117,7 +138,7 @@ public void TestQuotes() [TestMethod] public void TestDoubleQuotes() { - using CsvReader reader = new CsvReader(TestCSVFileNames[4]); + using CsvReader reader = new(TestCSVFileNames[4]); // first line reader.LoadLine(); //"A,B,C,D,E" @@ -135,7 +156,7 @@ public void TestDoubleQuotes() [TestMethod] public void TestLoadLineBool() { - using CsvReader reader = new CsvReader(TestCSVFileNames[1]); + using CsvReader reader = new(TestCSVFileNames[1]); Assert.AreEqual(1, reader.LineNumber); while (reader.LoadLine(out int columns)) { @@ -154,7 +175,7 @@ public void TestLoadLineBool() [TestMethod] public void TestNoEnterLastLine() { - using CsvReader reader = new CsvReader(TestCSVFileNames[3]); + using CsvReader reader = new(TestCSVFileNames[3]); Assert.AreEqual(1, reader.LineNumber); while (reader.LoadLine(out int columns)) { @@ -173,7 +194,7 @@ public void TestNoEnterLastLine() [TestMethod] public void TestLineReadValue() { - using CsvReader reader = new CsvReader(TestCSVFileNames[2]); + using CsvReader reader = new(TestCSVFileNames[2]); float lastColumnValue = float.NaN; while (reader.LoadLine(out int columns)) { @@ -188,7 +209,7 @@ public void TestLineReadValue() [TestMethod] public void TestNoEnterLastLineReadValue() { - using CsvReader reader = new CsvReader(TestCSVFileNames[3]); + using CsvReader reader = new(TestCSVFileNames[3]); float lastColumnValue = float.NaN; while (reader.LoadLine(out int columns)) { @@ -209,13 +230,64 @@ public void TestNoEnterLastLineReadValue() [TestMethod] public void TestReadingSpan() { - using CsvReader reader = new CsvReader(TestCSVFileNames[4]); + using CsvReader reader = new(TestCSVFileNames[4]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + } + +[TestMethod] + public void TestReadingUnixLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[6]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); + } + [TestMethod] + public void TestReadingWindowsLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[5]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); + } + + [TestMethod] + public void TestReadingLinuxLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[6]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); + } + + [TestMethod] + public void TestReadingMacLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[7]); //"\"abc\"\"1\",2,3,4,5" Assert.AreEqual(5, reader.Headers.Length); reader.LoadLine(out int columns); Assert.AreEqual(5, columns); reader.Get(out ReadOnlySpan span, 0); Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); } } }