Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix: cannot properly deserialize csv with mismatched number of columns #11

Merged
merged 3 commits into from
Jul 1, 2024
Merged

Fix: cannot properly deserialize csv with mismatched number of columns #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b160399
Fix cannot properly deserialize csv with mismatched number of columns
AnnulusGames Jul 1, 2024
7014c32
Add test
AnnulusGames Jul 1, 2024
b2df970
Fix when Key is string
AnnulusGames Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 117 additions & 37 deletions src/Csv.SourceGenerator/CsvSerializerGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ void EmitReadHeader()
{
builder.AppendLine("var allowComments = reader.Options.AllowComments;");
builder.AppendLine("if (allowComments) reader.TrySkipComment();");
builder.AppendLine("if (reader.Options.HasHeader) reader.AdvanceToEndOfLine();");
builder.AppendLine("if (reader.Options.HasHeader) reader.TrySkipLine();");
}

var members = type.Members;
Expand All @@ -391,36 +391,59 @@ void EmitReadHeader()
using (builder.BeginBlockScope("while (reader.Remaining > 0)"))
{
builder.AppendLine("if (reader.TryReadEndOfLine()) continue;");
builder.AppendLine("if (allowComments && reader.TrySkipComment()) continue;");
builder.AppendLine("if (allowComments && reader.TrySkipComment(false)) continue;");

EmitDeserializeLocalVariables(type, builder);
builder.AppendLine("var ___endOfLine = false;");

for (int i = 0; i < members.Count; i++)
{
EmitReadMember(members[i], builder);

if (i == members.Count - 1)
using (builder.BeginBlockScope($"for (int __i = 0; __i <= {members.Max(x => x.Index)}; __i++)"))
{
using (builder.BeginBlockScope("switch (__i)"))
{
builder.AppendLine();
builder.AppendLine("ADD_ITEM:");
builder.Append("list.Add(");
using (builder.BeginBlockScope("new()"))
foreach (var member in members)
{
foreach (var member in type.Members)
using (builder.BeginIndentScope($"case {member.Index}:"))
{
builder.AppendLine($"{member.Symbol.Name} = __{member.Symbol.Name},");
EmitReadMember(member, builder);
builder.AppendLine("break;");
}
}
builder.AppendLine(");");
using (builder.BeginIndentScope("default:"))
{
builder.AppendLine("reader.SkipField();");
builder.AppendLine("break;");
}
}
else

using (builder.BeginBlockScope("if (reader.TryReadEndOfLine(true))"))
{
builder.AppendLine("___endOfLine = true;");
builder.AppendLine("goto ADD_ITEM;");
}

builder.AppendLine("if (!reader.TryReadSeparator(false)) goto ADD_ITEM;");
}

builder.AppendLine();
builder.AppendLine("ADD_ITEM:");
builder.Append("list.Add(");
using (builder.BeginBlockScope("new()"))
{
foreach (var member in type.Members)
{
builder.AppendLine("if (reader.TryReadEndOfLine()) goto ADD_ITEM;");
builder.AppendLine("if (!reader.TryReadSeparator()) goto ADD_ITEM;");
builder.AppendLine($"{member.Symbol.Name} = __{member.Symbol.Name},");
}
}
builder.AppendLine(");");

builder.AppendLine();
using (builder.BeginBlockScope("if (!___endOfLine)"))
{
builder.AppendLine("if (!reader.TrySkipLine()) goto RETURN;");
}
}

builder.AppendLine("RETURN:");
builder.AppendLine("return list.AsSpan().ToArray();");
}

Expand All @@ -437,32 +460,55 @@ void EmitReadHeader()
builder.AppendLine("if (allowComments && reader.TrySkipComment(false)) continue;");

EmitDeserializeLocalVariables(type, builder);
builder.AppendLine("var ___endOfLine = false;");

for (int i = 0; i < members.Count; i++)
using (builder.BeginBlockScope($"for (int __i = 0; __i <= {members.Max(x => x.Index)}; __i++)"))
{
EmitReadMember(members[i], builder);

if (i == members.Count - 1)
using (builder.BeginBlockScope("switch (__i)"))
{
builder.AppendLine();
builder.AppendLine("ADD_ITEM:");
using (builder.BeginBlockScope("destination[n++] = new()"))
foreach (var member in members)
{
foreach (var member in type.Members)
using (builder.BeginIndentScope($"case {member.Index}:"))
{
builder.AppendLine($"{member.Symbol.Name} = __{member.Symbol.Name},");
EmitReadMember(member, builder);
builder.AppendLine("break;");
}
}
builder.AppendLine(";");
using (builder.BeginIndentScope("default:"))
{
builder.AppendLine("reader.SkipField();");
builder.AppendLine("break;");
}
}
else

using (builder.BeginBlockScope("if (reader.TryReadEndOfLine(true))"))
{
builder.AppendLine("if (reader.TryReadEndOfLine(true)) goto ADD_ITEM;");
builder.AppendLine("if (!reader.TryReadSeparator(false)) goto ADD_ITEM;");
builder.AppendLine("___endOfLine = true;");
builder.AppendLine("goto ADD_ITEM;");
}

builder.AppendLine("if (!reader.TryReadSeparator(false)) goto ADD_ITEM;");
}

builder.AppendLine();
builder.AppendLine("ADD_ITEM:");
using (builder.BeginBlockScope("destination[n++] = new()"))
{
foreach (var member in type.Members)
{
builder.AppendLine($"{member.Symbol.Name} = __{member.Symbol.Name},");
}
}
builder.AppendLine(";");

builder.AppendLine();
using (builder.BeginBlockScope("if (!___endOfLine)"))
{
builder.AppendLine("if (!reader.TrySkipLine()) goto RETURN;");
}
}

builder.AppendLine("RETURN:");
builder.AppendLine("return n;");
}
}
Expand All @@ -484,14 +530,22 @@ void EmitReadHeader()
builder.AppendLine("var keyBuffer = new global::Csv.Internal.TempList<byte>();");
using (builder.BeginBlockScope("try"))
{
builder.AppendLine("var ___endOfLine = false;");

using (builder.BeginBlockScope($"for (int i = 0; i < {members.Count}; i++)"))
{
builder.AppendLine("reader.ReadUtf8(ref keyBuffer);");
builder.AppendLine("map[i] = GetColumnIndex(keyBuffer.AsSpan());");
builder.AppendLine("keyBuffer.Clear(false);");
builder.AppendLine("if (reader.TryReadEndOfLine(true)) break;");
using (builder.BeginBlockScope("if (reader.TryReadEndOfLine(true))"))
{
builder.AppendLine("___endOfLine = true;");
builder.AppendLine("break;");
}
builder.AppendLine($"if (i != {members.Count} - 1) reader.TryReadSeparator(false);");
}

builder.AppendLine("if (!___endOfLine) reader.TrySkipLine();");
}
using (builder.BeginBlockScope("finally"))
{
Expand All @@ -513,10 +567,11 @@ void EmitReadHeader()
builder.AppendLine("if (allowComments && reader.TrySkipComment(false)) continue;");

EmitDeserializeLocalVariables(type, builder);
builder.AppendLine("var ___endOfLine = false;");

using (builder.BeginBlockScope("foreach (var index in map)"))
using (builder.BeginBlockScope("foreach (var ___i in map)"))
{
using (builder.BeginBlockScope("switch (index)"))
using (builder.BeginBlockScope("switch (___i)"))
{
for (int i = 0; i < members.Count; i++)
{
Expand All @@ -532,7 +587,12 @@ void EmitReadHeader()
}
}

builder.AppendLine("if (reader.TryReadEndOfLine(true)) goto ADD_ITEM;");
using (builder.BeginBlockScope("if (reader.TryReadEndOfLine(true))"))
{
builder.AppendLine("___endOfLine = true;");
builder.AppendLine("goto ADD_ITEM;");
}

builder.AppendLine("if (!reader.TryReadSeparator(false)) goto ADD_ITEM;");
}

Expand All @@ -547,8 +607,15 @@ void EmitReadHeader()
}
}
builder.AppendLine(");");

builder.AppendLine();
using (builder.BeginBlockScope("if (!___endOfLine)"))
{
builder.AppendLine("if (!reader.TrySkipLine()) goto RETURN;");
}
}

builder.AppendLine("RETURN:");
builder.AppendLine("return list.AsSpan().ToArray();");
}

Expand All @@ -565,10 +632,11 @@ void EmitReadHeader()
builder.AppendLine("if (allowComments && reader.TrySkipComment(false)) continue;");

EmitDeserializeLocalVariables(type, builder);
builder.AppendLine("var ___endOfLine = false;");

using (builder.BeginBlockScope("foreach (var index in map)"))
using (builder.BeginBlockScope("foreach (var __i in map)"))
{
using (builder.BeginBlockScope("switch (index)"))
using (builder.BeginBlockScope("switch (__i)"))
{
for (int i = 0; i < members.Count; i++)
{
Expand All @@ -584,7 +652,12 @@ void EmitReadHeader()
}
}

builder.AppendLine("if (reader.TryReadEndOfLine(true)) goto ADD_ITEM;");
using (builder.BeginBlockScope("if (reader.TryReadEndOfLine(true))"))
{
builder.AppendLine("___endOfLine = true;");
builder.AppendLine("goto ADD_ITEM;");
}

builder.AppendLine("if (!reader.TryReadSeparator(false)) goto ADD_ITEM;");
}

Expand All @@ -598,8 +671,15 @@ void EmitReadHeader()
}
}
builder.AppendLine(";");

builder.AppendLine();
using (builder.BeginBlockScope("if (!___endOfLine)"))
{
builder.AppendLine("if (!reader.TrySkipLine()) goto RETURN;");
}
}

builder.AppendLine("RETURN:");
builder.AppendLine("return n;");
}
}
Expand Down
28 changes: 8 additions & 20 deletions src/Csv/CsvReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ public char ReadChar()
}
}

internal int SkipField()
public int SkipField()
{
if (IsNextSeparatorOrNewline())
{
Expand Down Expand Up @@ -279,38 +279,26 @@ public bool TrySkipComment(bool skipWhiteSpace = true)
{
if (skipWhiteSpace) SkipWhitespace();
var isComment = reader.IsNext((byte)'#', true);
if (isComment) AdvanceToEndOfLine();
if (isComment) TrySkipLine();

return isComment;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AdvanceToEndOfLine()
public bool TrySkipLine()
{
if (!reader.TryAdvanceToAny([(byte)'\n', (byte)'\r'], false)) return;
reader.TryRead(out var c1);
if (c1 == '\r' && reader.TryPeek(out var c2) && c2 == '\n')
{
reader.Advance(1);
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AdvanceToSeparatorOrEndOfLine()
{
SkipWhitespace();

while (true)
if (!reader.TryAdvanceToAny([(byte)'\n', (byte)'\r'], false))
{
if (!reader.TryPeek(out var c)) return;
if (c == (byte)'\n' || c == (byte)'\r' || c == separator) break;
return false;
}

reader.TryRead(out var c1);
if (c1 == '\r' && reader.TryPeek(out var c2) && c2 == '\n')
{
reader.Advance(1);
}

return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down
19 changes: 19 additions & 0 deletions tests/Csv.Tests/SerializeTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,25 @@ public void Test_Deserialize_Complex()

CollectionAssert.AreEqual(expected, actual);
}

[Test]
public void Test_Deserialize_ExtraColumns()
{
var csv =
@"Name,Age,Dummy1,Dummy2
Alex,21,1,""a""
Bob,35,25,""b""
Charles,17,23,""c"""u8;

User[] actual = CsvSerializer.Deserialize<User>(new ReadOnlySequence<byte>(csv.ToArray()));
User[] expected = [
new() { Name = "Alex", Age = 21 },
new() { Name = "Bob", Age = 35 },
new() { Name = "Charles", Age = 17 }
];

CollectionAssert.AreEqual(expected, actual);
}
}

[CsvObject]
Expand Down