diff --git a/src/TMG-Framework/Utilities/CSVReader.cs b/src/TMG-Framework/Utilities/CSVReader.cs index d161659..c89eaaa 100644 --- a/src/TMG-Framework/Utilities/CSVReader.cs +++ b/src/TMG-Framework/Utilities/CSVReader.cs @@ -235,12 +235,13 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); } columns = numberOfColumns; - return true; + return columns > 0; } } c = DataBuffer[DataBufferPosition++]; - if ((c == '\n') || (c == '\0')) + if ((prevC == '\r' && c != '\n' && c != '\0') || (c == '\n') || (c == '\0')) { + // Create a new column if this is a Linux file format. if (prevC != '\r') { if (Data.Length <= numberOfColumns) @@ -250,6 +251,12 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); addOne = false; } + else if (prevC == '\r' && c != '\n' && c != '\0') + { + // If it was just a MAC encoded line ending then we need to process the current character in the next load. + DataBufferPosition--; + addOne = false; + } break; } if (c == '"') @@ -322,12 +329,13 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); } columns = numberOfColumns; - return true; + return columns > 0; } } c = DataBuffer[DataBufferPosition++]; - if ((c == '\n') || (c == '\0')) + if ((prevC == '\r' && c != '\n' && c != '\0') || (c == '\n') || (c == '\0')) { + // Create a new column if this is a Linux file format. if (prevC != '\r') { if (Data.Length <= numberOfColumns) @@ -337,6 +345,12 @@ public bool LoadLine(out int columns) Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i); addOne = false; } + else if (prevC == '\r' && c != '\n' && c != '\0') + { + // If it was just a MAC encoded line ending then we need to process the current character in the next load. + DataBufferPosition--; + addOne = false; + } break; } if (c == '"') diff --git a/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs b/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs index 1336e3d..6215c20 100644 --- a/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs +++ b/tests/TMG-Framework.Test/Utilities/TestCSVReader.cs @@ -31,46 +31,67 @@ namespace XTMF.Testing.TMG.Data [TestClass] public class TestCSVReader { - private readonly string[] TestCSVFileNames = new[] { "CSVTest1.csv", "CSVTest2.csv", "CSVTest3.csv", "CSVTest4.csv", "CSVTest5.csv", "CSVTest6.csv" }; + private readonly string[] TestCSVFileNames = new[] { "CSVTest1.csv", "CSVTest2.csv", "CSVTest3.csv", "CSVTest4.csv", "CSVTest5.csv", "CSVTest6.csv", "CSVTest7.csv", "CSVTest8.csv" }; [TestInitialize] public void CreateTestEnvironment() { if (!IsEnvironmentLoaded()) { - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[0])) + using (StreamWriter writer = new(TestCSVFileNames[0])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("1,2,3,4,5"); writer.WriteLine("3,1,4,5,2"); writer.WriteLine("1.23,4.56,7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[1])) + using (StreamWriter writer = new(TestCSVFileNames[1])) { writer.WriteLine("\"A\",\"B\",\"C\",\"D\",\"E\""); writer.WriteLine("\"1\",\"2\",3,\"4\",5"); writer.WriteLine("3,1,\"4\",5,2"); writer.WriteLine("1.23,\"4.56\",7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[2])) + using (StreamWriter writer = new(TestCSVFileNames[2])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("1,2,3,4,5"); writer.WriteLine("3,1,4,5,2"); writer.WriteLine("1.23,4.56,7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[3])) + using (StreamWriter writer = new(TestCSVFileNames[3])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("1,2,3,4,5"); writer.WriteLine("3,1,4,5,2"); writer.Write("1.23,4.56,7.89,10.1112,0.1314"); } - using (StreamWriter writer = new StreamWriter(TestCSVFileNames[4])) + using (StreamWriter writer = new(TestCSVFileNames[4])) { writer.WriteLine("A,B,C,D,E"); writer.WriteLine("\"abc\"\"1\",2,3,4,5"); } + + // The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Windows format. + using (StreamWriter writer = new(TestCSVFileNames[5])) + { + writer.Write("A,B,C,D,E\r\n"); + writer.Write("\"abc\"\"1\",2,3,4,5\r\n"); + } + + // The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Linux format. + using (StreamWriter writer = new(TestCSVFileNames[6])) + { + writer.Write("A,B,C,D,E\n"); + writer.Write("\"abc\"\"1\",2,3,4,5\n"); + } + + // The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Mac format. + using (StreamWriter writer = new(TestCSVFileNames[7])) + { + writer.Write("A,B,C,D,E\r"); + writer.Write("\"abc\"\"1\",2,3,4,5\r"); + } } } @@ -86,7 +107,7 @@ public bool IsEnvironmentLoaded() [TestMethod] public void TestCSVHeaders() { - using CsvReader reader = new CsvReader(TestCSVFileNames[0]); + using CsvReader reader = new(TestCSVFileNames[0]); //"A,B,C,D,E" var headers = reader.Headers; for (int i = 0; i < 5; i++) @@ -98,7 +119,7 @@ public void TestCSVHeaders() [TestMethod] public void TestQuotes() { - using CsvReader reader = new CsvReader(TestCSVFileNames[1]); + using CsvReader reader = new(TestCSVFileNames[1]); //"A,B,C,D,E" var headers = reader.Headers; for (int i = 0; i < 5; i++) @@ -117,7 +138,7 @@ public void TestQuotes() [TestMethod] public void TestDoubleQuotes() { - using CsvReader reader = new CsvReader(TestCSVFileNames[4]); + using CsvReader reader = new(TestCSVFileNames[4]); // first line reader.LoadLine(); //"A,B,C,D,E" @@ -135,7 +156,7 @@ public void TestDoubleQuotes() [TestMethod] public void TestLoadLineBool() { - using CsvReader reader = new CsvReader(TestCSVFileNames[1]); + using CsvReader reader = new(TestCSVFileNames[1]); Assert.AreEqual(1, reader.LineNumber); while (reader.LoadLine(out int columns)) { @@ -154,7 +175,7 @@ public void TestLoadLineBool() [TestMethod] public void TestNoEnterLastLine() { - using CsvReader reader = new CsvReader(TestCSVFileNames[3]); + using CsvReader reader = new(TestCSVFileNames[3]); Assert.AreEqual(1, reader.LineNumber); while (reader.LoadLine(out int columns)) { @@ -173,7 +194,7 @@ public void TestNoEnterLastLine() [TestMethod] public void TestLineReadValue() { - using CsvReader reader = new CsvReader(TestCSVFileNames[2]); + using CsvReader reader = new(TestCSVFileNames[2]); float lastColumnValue = float.NaN; while (reader.LoadLine(out int columns)) { @@ -188,7 +209,7 @@ public void TestLineReadValue() [TestMethod] public void TestNoEnterLastLineReadValue() { - using CsvReader reader = new CsvReader(TestCSVFileNames[3]); + using CsvReader reader = new(TestCSVFileNames[3]); float lastColumnValue = float.NaN; while (reader.LoadLine(out int columns)) { @@ -209,13 +230,64 @@ public void TestNoEnterLastLineReadValue() [TestMethod] public void TestReadingSpan() { - using CsvReader reader = new CsvReader(TestCSVFileNames[4]); + using CsvReader reader = new(TestCSVFileNames[4]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + } + +[TestMethod] + public void TestReadingUnixLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[6]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); + } + [TestMethod] + public void TestReadingWindowsLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[5]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); + } + + [TestMethod] + public void TestReadingLinuxLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[6]); + //"\"abc\"\"1\",2,3,4,5" + Assert.AreEqual(5, reader.Headers.Length); + reader.LoadLine(out int columns); + Assert.AreEqual(5, columns); + reader.Get(out ReadOnlySpan span, 0); + Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); + } + + [TestMethod] + public void TestReadingMacLineEndings() + { + using CsvReader reader = new(TestCSVFileNames[7]); //"\"abc\"\"1\",2,3,4,5" Assert.AreEqual(5, reader.Headers.Length); reader.LoadLine(out int columns); Assert.AreEqual(5, columns); reader.Get(out ReadOnlySpan span, 0); Assert.AreEqual("abc\"1", new string(span)); + Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns."); } } }