diff --git a/README.md b/README.md index 8fa108af..74d1c8e3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ All methods accept the following `options` * `objectMode=true`: Ensure that `data` events have an object emitted rather than the stringified version set to false to have a stringified buffer. * `headers=false`: Ste to true if you expect the first line of your `CSV` to contain headers, alternatly you can specify an array of headers to use. * `ignoreEmpty=false`: If you wish to ignore empty rows. +* `discardUnmappedColumns=false`: If you want to discard columns that do not map to a header. * `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`. * **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimiter * `quote='"'`: The character to use to escape values that contain a delimiter. @@ -473,7 +474,7 @@ var formatStream = csv emailAddress: obj.Email_Address, verified: obj.Verified }; - }); + }); csv .fromPath("in.csv", {headers: true}) .pipe(formatStream) diff --git a/lib/parser_stream.js b/lib/parser_stream.js index dc18af08..69d503f9 100644 --- a/lib/parser_stream.js +++ b/lib/parser_stream.js @@ -51,6 +51,7 @@ function ParserStream(options) { this.parser = createParser(options); this._headers = options.headers; this._ignoreEmpty = options.ignoreEmpty; + this._discardUnmappedColumns = options.discardUnmappedColumns; this.__objectMode = options.objectMode; this.__buffered = []; return this; @@ -111,6 +112,7 @@ extended(ParserStream).extend({ rows = data.rows; if (!this._parsedHeaders) { var headers = this._headers; + var discardUnmappedColumns = this._discardUnmappedColumns; if (extended.isBoolean(headers) && headers) { headers = this.__handleLine(rows.shift(), 0, true); } @@ -120,7 +122,11 @@ extended(ParserStream).extend({ this.__transform = function (data, index) { var ret = {}, i = -1, val; if (data.length > headersLength) { - self.emit("error", new Error("Unexpected Error: column header mismatch expected: " + headersLength + " columns got: " + data.length)); + if (discardUnmappedColumns) { + data.splice(headersLength); + } else { + self.emit("error", new Error("Unexpected Error: column header mismatch expected: " + headersLength + " columns got: " + data.length)); + } } while (++i < headersLength) { val = data[i]; @@ -268,4 +274,4 @@ extended(ParserStream).extend({ }); -module.exports = ParserStream; \ No newline at end of file +module.exports = ParserStream; diff --git a/test/assets/test23.csv b/test/assets/test23.csv new file mode 100644 index 00000000..b04cfd68 --- /dev/null +++ b/test/assets/test23.csv @@ -0,0 +1,4 @@ +first_name,last_name,email_address +First1,Last1,email1@email.com +First2,Last2,email2@email.com,Unmapped1 +First3,Last3,email3@email.com,Unmapped1,Unmapped2 \ No newline at end of file diff --git a/test/fast-csv.test.js b/test/fast-csv.test.js index 90db78e0..ef879352 100644 --- a/test/fast-csv.test.js +++ b/test/fast-csv.test.js @@ -141,8 +141,14 @@ var expected14 = [ var expected21 = [ {"first_name": "First\n1", "last_name": "Last\n1", "email_address": "email1@email.com", address: "1 Street St,\nState ST, 88888"}, - {"first_name": "First\n2", "last_name": "Last\n2", "email_address": "email2@email.com", address: "2 Street St,\nState ST, 88888"}, -] + {"first_name": "First\n2", "last_name": "Last\n2", "email_address": "email2@email.com", address: "2 Street St,\nState ST, 88888"} +]; + +var expected23 = [ + {"first_name": "First1", "last_name": "Last1", "email_address": "email1@email.com"}, + {"first_name": "First2", "last_name": "Last2", "email_address": "email2@email.com"}, + {"first_name": "First3", "last_name": "Last3", "email_address": "email3@email.com"} +]; it.describe("fast-csv", function (it) { @@ -461,6 +467,21 @@ it.describe("fast-csv", function (it) { }); }); + it.should("discard extra columns that do not map to a header with discardUnmappedColumns option", function (next) { + var actual = []; + csv + .fromPath(path.resolve(__dirname, "./assets/test23.csv"), {headers: true, discardUnmappedColumns: true}) + .on("record", function (data, index) { + actual.push(data); + }) + .on("error", next) + .on("end", function (count) { + assert.deepEqual(actual, expected23); + assert.equal(count, actual.length); + next(); + }); + }); + it.describe("alternate delimiters", function (it) { it.should("support tab delimiters", function (next) { var actual = []; @@ -1000,4 +1021,4 @@ it.describe("fast-csv", function (it) { }); }); }); -}); \ No newline at end of file +});