Merge pull request #170 from pietercolpaert/fix-131

Fix #131
C2FO · Jan 30, 2017 · f8e6a4b · f8e6a4b
2 parents 8e9e823 + b43d85c
commit f8e6a4b
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 1 deletion.
diff --git a/lib/parser/parser_stream.js b/lib/parser/parser_stream.js
@@ -194,11 +194,21 @@ extended(ParserStream).extend({
         }
     },
 
+    __removeBOM: function (data) {
+        // Catches EFBBBF (UTF-8 BOM) because the buffer-to-string
+        // conversion translates it to FEFF (UTF-16 BOM)
+        if (data && typeof data == 'string' && data.charCodeAt(0) == '0xFEFF') {
+	    return data.slice(1);
+	}
+	return data;
+    },
+
     _transform: function (data, encoding, done) {
         var lines = this.lines,
             lineData = (lines + this.decoder.write(data)),
             self = this;
         if (lineData.length > 1) {
+            lineData = this.__removeBOM(lineData);
             this._parse(lineData, true, function (err, lineData) {
                 if (err) {
                     done(err);

diff --git a/test/assets/test28.csv b/test/assets/test28.csv
@@ -0,0 +1,10 @@
+first_name,last_name,email_address,address
+First1,Last1,[email protected],"1 Street St, State ST, 88888"
+First2,Last2,[email protected],"2 Street St, State ST, 88888"
+First3,Last3,[email protected],"3 Street St, State ST, 88888"
+First4,Last4,[email protected],"4 Street St, State ST, 88888"
+First5,Last5,[email protected],"5 Street St, State ST, 88888"
+First6,Last6,[email protected],"6 Street St, State ST, 88888"
+First7,Last7,[email protected],"7 Street St, State ST, 88888"
+First8,Last8,[email protected],"8 Street St, State ST, 88888"
+First9,Last9,[email protected],"9 Street St, State ST, 88888"
diff --git a/test/issues.test.js b/test/issues.test.js
@@ -283,4 +283,21 @@ it.describe("github issues", function (it) {
         });
 
     });
-});
+
+    it.describe("#131", function (it) {
+
+        it.should("parse a csv with a UTF-8 Byte Order Mark", function (next) {
+            var actual = [];
+            csv
+              .fromPath(path.resolve(__dirname, "./assets/test28.csv"), {headers: true})
+              .on("data", function (data, index) {
+                  actual.push(data);
+              }).
+              on("end", function (count) {
+                  assert.deepEqual(actual[0]["first_name"], "First1");
+                  assert.equal(count, actual.length);
+                  next();
+              });
+        });
+    });
+});