Skip to content

Update CSVReader to work with Linux and Windows Line endings #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JamesVaughan merged 1 commit into from
Aug 10, 2026
Merged

Update CSVReader to work with Linux and Windows Line endings #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions src/TMG-Framework/Utilities/CSVReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,12 +235,13 @@ public bool LoadLine(out int columns)
Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i);
}
columns = numberOfColumns;
return true;
return columns > 0;
}
}
c = DataBuffer[DataBufferPosition++];
if ((c == '\n') || (c == '\0'))
if ((prevC == '\r' && c != '\n' && c != '\0') || (c == '\n') || (c == '\0'))
{
// Create a new column if this is a Linux file format.
if (prevC != '\r')
{
if (Data.Length <= numberOfColumns)
Expand All @@ -250,6 +251,12 @@ public bool LoadLine(out int columns)
Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i);
addOne = false;
}
else if (prevC == '\r' && c != '\n' && c != '\0')
{
// If it was just a MAC encoded line ending then we need to process the current character in the next load.
DataBufferPosition--;
addOne = false;
}
break;
}
if (c == '"')
Expand Down Expand Up @@ -322,12 +329,13 @@ public bool LoadLine(out int columns)
Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i);
}
columns = numberOfColumns;
return true;
return columns > 0;
}
}
c = DataBuffer[DataBufferPosition++];
if ((c == '\n') || (c == '\0'))
if ((prevC == '\r' && c != '\n' && c != '\0') || (c == '\n') || (c == '\0'))
{
// Create a new column if this is a Linux file format.
if (prevC != '\r')
{
if (Data.Length <= numberOfColumns)
Expand All @@ -337,6 +345,12 @@ public bool LoadLine(out int columns)
Data[numberOfColumns++] = new CsvPartition(prevEnd + 1, i);
addOne = false;
}
else if (prevC == '\r' && c != '\n' && c != '\0')
{
// If it was just a MAC encoded line ending then we need to process the current character in the next load.
DataBufferPosition--;
addOne = false;
}
break;
}
if (c == '"')
Expand Down
100 changes: 86 additions & 14 deletions tests/TMG-Framework.Test/Utilities/TestCSVReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,46 +31,67 @@ namespace XTMF.Testing.TMG.Data
[TestClass]
public class TestCSVReader
{
private readonly string[] TestCSVFileNames = new[] { "CSVTest1.csv", "CSVTest2.csv", "CSVTest3.csv", "CSVTest4.csv", "CSVTest5.csv", "CSVTest6.csv" };
private readonly string[] TestCSVFileNames = new[] { "CSVTest1.csv", "CSVTest2.csv", "CSVTest3.csv", "CSVTest4.csv", "CSVTest5.csv", "CSVTest6.csv", "CSVTest7.csv", "CSVTest8.csv" };

[TestInitialize]
public void CreateTestEnvironment()
{
if (!IsEnvironmentLoaded())
{
using (StreamWriter writer = new StreamWriter(TestCSVFileNames[0]))
using (StreamWriter writer = new(TestCSVFileNames[0]))
{
writer.WriteLine("A,B,C,D,E");
writer.WriteLine("1,2,3,4,5");
writer.WriteLine("3,1,4,5,2");
writer.WriteLine("1.23,4.56,7.89,10.1112,0.1314");
}
using (StreamWriter writer = new StreamWriter(TestCSVFileNames[1]))
using (StreamWriter writer = new(TestCSVFileNames[1]))
{
writer.WriteLine("\"A\",\"B\",\"C\",\"D\",\"E\"");
writer.WriteLine("\"1\",\"2\",3,\"4\",5");
writer.WriteLine("3,1,\"4\",5,2");
writer.WriteLine("1.23,\"4.56\",7.89,10.1112,0.1314");
}
using (StreamWriter writer = new StreamWriter(TestCSVFileNames[2]))
using (StreamWriter writer = new(TestCSVFileNames[2]))
{
writer.WriteLine("A,B,C,D,E");
writer.WriteLine("1,2,3,4,5");
writer.WriteLine("3,1,4,5,2");
writer.WriteLine("1.23,4.56,7.89,10.1112,0.1314");
}
using (StreamWriter writer = new StreamWriter(TestCSVFileNames[3]))
using (StreamWriter writer = new(TestCSVFileNames[3]))
{
writer.WriteLine("A,B,C,D,E");
writer.WriteLine("1,2,3,4,5");
writer.WriteLine("3,1,4,5,2");
writer.Write("1.23,4.56,7.89,10.1112,0.1314");
}
using (StreamWriter writer = new StreamWriter(TestCSVFileNames[4]))
using (StreamWriter writer = new(TestCSVFileNames[4]))
{
writer.WriteLine("A,B,C,D,E");
writer.WriteLine("\"abc\"\"1\",2,3,4,5");
}

// The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Windows format.
using (StreamWriter writer = new(TestCSVFileNames[5]))
{
writer.Write("A,B,C,D,E\r\n");
writer.Write("\"abc\"\"1\",2,3,4,5\r\n");
}

// The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Linux format.
using (StreamWriter writer = new(TestCSVFileNames[6]))
{
writer.Write("A,B,C,D,E\n");
writer.Write("\"abc\"\"1\",2,3,4,5\n");
}

// The smae as TestCSVFileNames[4] but forcing the line breaks to be in the Mac format.
using (StreamWriter writer = new(TestCSVFileNames[7]))
{
writer.Write("A,B,C,D,E\r");
writer.Write("\"abc\"\"1\",2,3,4,5\r");
}
}
}

Expand All @@ -86,7 +107,7 @@ public bool IsEnvironmentLoaded()
[TestMethod]
public void TestCSVHeaders()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[0]);
using CsvReader reader = new(TestCSVFileNames[0]);
//"A,B,C,D,E"
var headers = reader.Headers;
for (int i = 0; i < 5; i++)
Expand All @@ -98,7 +119,7 @@ public void TestCSVHeaders()
[TestMethod]
public void TestQuotes()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[1]);
using CsvReader reader = new(TestCSVFileNames[1]);
//"A,B,C,D,E"
var headers = reader.Headers;
for (int i = 0; i < 5; i++)
Expand All @@ -117,7 +138,7 @@ public void TestQuotes()
[TestMethod]
public void TestDoubleQuotes()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[4]);
using CsvReader reader = new(TestCSVFileNames[4]);
// first line
reader.LoadLine();
//"A,B,C,D,E"
Expand All @@ -135,7 +156,7 @@ public void TestDoubleQuotes()
[TestMethod]
public void TestLoadLineBool()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[1]);
using CsvReader reader = new(TestCSVFileNames[1]);
Assert.AreEqual(1, reader.LineNumber);
while (reader.LoadLine(out int columns))
{
Expand All @@ -154,7 +175,7 @@ public void TestLoadLineBool()
[TestMethod]
public void TestNoEnterLastLine()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[3]);
using CsvReader reader = new(TestCSVFileNames[3]);
Assert.AreEqual(1, reader.LineNumber);
while (reader.LoadLine(out int columns))
{
Expand All @@ -173,7 +194,7 @@ public void TestNoEnterLastLine()
[TestMethod]
public void TestLineReadValue()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[2]);
using CsvReader reader = new(TestCSVFileNames[2]);
float lastColumnValue = float.NaN;
while (reader.LoadLine(out int columns))
{
Expand All @@ -188,7 +209,7 @@ public void TestLineReadValue()
[TestMethod]
public void TestNoEnterLastLineReadValue()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[3]);
using CsvReader reader = new(TestCSVFileNames[3]);
float lastColumnValue = float.NaN;
while (reader.LoadLine(out int columns))
{
Expand All @@ -209,13 +230,64 @@ public void TestNoEnterLastLineReadValue()
[TestMethod]
public void TestReadingSpan()
{
using CsvReader reader = new CsvReader(TestCSVFileNames[4]);
using CsvReader reader = new(TestCSVFileNames[4]);
//"\"abc\"\"1\",2,3,4,5"
Assert.AreEqual(5, reader.Headers.Length);
reader.LoadLine(out int columns);
Assert.AreEqual(5, columns);
reader.Get(out ReadOnlySpan<char> span, 0);
Assert.AreEqual("abc\"1", new string(span));
}

[TestMethod]
public void TestReadingUnixLineEndings()
{
using CsvReader reader = new(TestCSVFileNames[6]);
//"\"abc\"\"1\",2,3,4,5"
Assert.AreEqual(5, reader.Headers.Length);
reader.LoadLine(out int columns);
Assert.AreEqual(5, columns);
reader.Get(out ReadOnlySpan<char> span, 0);
Assert.AreEqual("abc\"1", new string(span));
Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns.");
}
[TestMethod]
public void TestReadingWindowsLineEndings()
{
using CsvReader reader = new(TestCSVFileNames[5]);
//"\"abc\"\"1\",2,3,4,5"
Assert.AreEqual(5, reader.Headers.Length);
reader.LoadLine(out int columns);
Assert.AreEqual(5, columns);
reader.Get(out ReadOnlySpan<char> span, 0);
Assert.AreEqual("abc\"1", new string(span));
Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns.");
}

[TestMethod]
public void TestReadingLinuxLineEndings()
{
using CsvReader reader = new(TestCSVFileNames[6]);
//"\"abc\"\"1\",2,3,4,5"
Assert.AreEqual(5, reader.Headers.Length);
reader.LoadLine(out int columns);
Assert.AreEqual(5, columns);
reader.Get(out ReadOnlySpan<char> span, 0);
Assert.AreEqual("abc\"1", new string(span));
Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns.");
}

[TestMethod]
public void TestReadingMacLineEndings()
{
using CsvReader reader = new(TestCSVFileNames[7]);
//"\"abc\"\"1\",2,3,4,5"
Assert.AreEqual(5, reader.Headers.Length);
reader.LoadLine(out int columns);
Assert.AreEqual(5, columns);
reader.Get(out ReadOnlySpan<char> span, 0);
Assert.AreEqual("abc\"1", new string(span));
Assert.IsFalse(reader.LoadLine(out columns), $"There is an extra line at the end of the file with {columns} columns.");
}
}
}
Loading