Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Handle split CRLF #156

Closed
wants to merge 4 commits into from
Closed

Handle split CRLF #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
16b7a8e
Test issue 150
Sep 21, 2016
43c6d6a
Test split CRLF
Sep 20, 2016
8d63d59
Test ambiguous CR
Sep 20, 2016
0b9b82b
Keep the line if a new line is ambiguous
Sep 21, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions lib/parser/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,11 @@ function createParser(options) {
}

function getNextToken(line, cursor) {
var token, nextIndex, subStr = line.substr(cursor);
var token, tokenLen, nextIndex, subStr = line.substr(cursor);
if ((nextIndex = subStr.search(NEXT_TOKEN_REGEXP)) !== -1) {
token = line[cursor += nextIndex];
cursor += subStr.match(NEXT_TOKEN_REGEXP)[1].length - 1;
tokenLen = subStr.match(NEXT_TOKEN_REGEXP)[1].length;
token = line.substr(cursor + nextIndex, tokenLen);
cursor += nextIndex + tokenLen - 1;
}
return {token: token, cursor: cursor};
}
Expand All @@ -167,6 +168,11 @@ function createParser(options) {
items = [];
lastLineI = i;
} else {
// if ends with CR and there is more data, keep unparsed due to possible coming LF in CRLF
if (token === '\r' && hasMoreData) {
i = lastLineI;
cursor = null;
}
break;
}
} else if (hasComments && token === COMMENT) {
Expand Down
12 changes: 12 additions & 0 deletions test/issues.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,4 +239,16 @@ it.describe("github issues", function (it) {
});
});
});

it.describe("#150", function (it) {
it.should("not parse a row if a new line is ambiguous and there is more data", function () {
var data = "first_name,last_name,email_address\r";
var myParser = parser({delimiter: ","});
var parsedData = myParser(data, true);
assert.deepEqual(parsedData, {
"line": "first_name,last_name,email_address\r",
"rows": []
});
});
});
});
32 changes: 19 additions & 13 deletions test/parser.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ it.describe("fast-csv parser", function (it) {
});

it.should("parse a block of CSV text with a trailing delimiter", function () {
var data = "first_name,last_name,email_address,empty\nFirst1,Last1,[email protected],\n";
var data = "first_name,last_name,email_address,empty\rFirst1,Last1,[email protected],\r";
var myParser = parser({delimiter: ","});
assert.deepEqual(myParser(data, false), {
"line": "", "rows": [
Expand All @@ -330,7 +330,7 @@ it.describe("fast-csv parser", function (it) {
});

it.should("parse a block of CSV text with a trailing delimiter followed by a space", function() {
var data = "first_name,last_name,email_address,empty\nFirst1,Last1,[email protected], \n";
var data = "first_name,last_name,email_address,empty\nFirst1,Last1,[email protected], \r";
var myParser = parser({ delimiter: "," });
assert.deepEqual(myParser(data, false), {
"line": "", "rows": [
Expand All @@ -341,7 +341,7 @@ it.describe("fast-csv parser", function (it) {
});

it.should("parse a block of Space Separated Value text with a trailing delimiter", function() {
var data = "first_name last_name email_address empty\nFirst1 Last1 [email protected] \n";
var data = "first_name last_name email_address empty\rFirst1 Last1 [email protected] \r";
var myParser = parser({ delimiter: " " });
assert.deepEqual(myParser(data, false), {
"line": "", "rows": [
Expand Down Expand Up @@ -400,15 +400,13 @@ it.describe("fast-csv parser", function (it) {
});
});

it.should("parse a row if a new line is found and there is more data", function () {
it.should("not parse a row if an ambiguous new line is found and there is more data", function () {
var data = "first_name,last_name,email_address\r";
var myParser = parser({delimiter: ","});
var parsedData = myParser(data, true);
assert.deepEqual(parsedData, {
"line": "",
"rows": [
["first_name", "last_name", "email_address"]
]
"line": "first_name,last_name,email_address\r",
"rows": []
});
});

Expand Down Expand Up @@ -532,15 +530,13 @@ it.describe("fast-csv parser", function (it) {
});
});

it.should("parse a row if a new line is found and there is more data", function () {
it.should("not parse a row if an ambiguous new line is found and there is more data", function () {
var data = '"first_name","last_name","email_address"\r';
var myParser = parser({delimiter: ","});
var parsedData = myParser(data, true);
assert.deepEqual(parsedData, {
"line": "",
"rows": [
["first_name", "last_name", "email_address"]
]
"line": '"first_name","last_name","email_address"\r',
"rows": []
});
});
});
Expand Down Expand Up @@ -614,6 +610,16 @@ it.describe("fast-csv parser", function (it) {
});
});

it.should("not parse a row if a new line is incomplete and there is more data", function () {
var data = "first_name,last_name,email_address\r";
var myParser = parser({delimiter: ","});
var parsedData = myParser(data, true);
assert.deepEqual(parsedData, {
"line": "first_name,last_name,email_address\r",
"rows": []
});
});

it.should("not parse a row if there is a trailing delimiter and there is more data", function () {
var data = "first_name,last_name,email_address,";
var myParser = parser({delimiter: ","});
Expand Down