From f0a0e85d41bf5f6521e9f7a708c496bbe497ebfd Mon Sep 17 00:00:00 2001 From: Doug Martin Date: Fri, 13 Mar 2015 09:22:02 -0500 Subject: [PATCH] v0.6.0 * Removed try catch from emit to allow bubbling up of errors to process, if one is thrown #93 * This also fixed issue #92 where a loop was entered when `this.emit("error")` was called. * Added new tests --- History.md | 6 + docs-md/coverage.html | 622 - docs/History.html | 8 + lib/parser/parser_stream.js | 32 +- package.json | 2 +- .../{issue68.csv => issue68-invalid.tsv} | 2 +- test/assets/issue68.tsv | 20001 ++++++++++++++++ test/assets/issue93.csv | 20001 ++++++++++++++++ test/issues.test.js | 89 +- 9 files changed, 40105 insertions(+), 658 deletions(-) delete mode 100644 docs-md/coverage.html rename test/assets/{issue68.csv => issue68-invalid.tsv} (99%) create mode 100644 test/assets/issue68.tsv create mode 100644 test/assets/issue93.csv diff --git a/History.md b/History.md index 243f48a3..f1dff75d 100644 --- a/History.md +++ b/History.md @@ -1,3 +1,9 @@ +# v0.6.0 + +* Removed try catch from emit to allow bubbling up of errors to process, if one is thrown [#93](https://github.com/C2FO/fast-csv/issues/93) + * This also fixed issue [#92](https://github.com/C2FO/fast-csv/issues/92) where a loop was entered when `this.emit("error")` was called. +* Added new tests + # v0.5.7 * Strict record handling [#53](https://github.com/C2FO/fast-csv/pull/53) - [@derjust](https://github.com/derjust) diff --git a/docs-md/coverage.html b/docs-md/coverage.html deleted file mode 100644 index d3bfc6eb..00000000 --- a/docs-md/coverage.html +++ /dev/null @@ -1,622 +0,0 @@ - - - - - -Test Coverage - - - -
- - - - - -
-
Overview
- 		-
- Coverage98.90 - SLOC323 - LOC91 - Missed1 -
-
-
-
- -
- - - - - -
-
index.js
- 		-
- Coverage98.90 - SLOC323 - LOC91 - Missed1 -
-
- 
  1.  /**
    2. 
-
  2.   * @projectName fast-csv
    3. 
-
  3.   * @github https://github.com/C2FO/fast-csv
    4. 
-
  4.   * @includeDoc [Test Coverage] [../docs-md/coverage.html]
    5. 
-
  5.   * @header
    6. 
-
  6.   *
    7. 
-
  7.   * [![build status](https://secure.travis-ci.org/C2FO/fast-csv.png)](http://travis-ci.org/C2FO/fast-csv)
    8. 
-
  8.   * # Fast-csv
    9. 
-
  9.   *
    10. 
-
  10.   * This is a library is aimed at providing fast CSV parsing. It accomplishes this by not handling some of the more complex
    11. 
-
  11.   * edge cases such as multi line rows. However it does support escaped values, embedded commas, double and single quotes.
    12. 
-
  12.   *
    13. 
-
  13.   * ## Installation
    14. 
-
  14.   *
    15. 
-
  15.   * `npm install fast-csv`
    16. 
-
  16.   *
    17. 
-
  17.   * ## Usage
    18. 
-
  18.   *
    19. 
-
  19.   * To parse a file.
    20. 
-
  20.   *
    21. 
-
  21.   * ```javascript
    22. 
-
  22.   * var csv = require("fast-csv");
    23. 
-
  23.   *
    24. 
-
  24.   * csv("my.csv")
    25. 
-
  25.   *  .on("data", function(data){
    26. 
-
  26.   *      console.log(data):
    27. 
-
  27.   *  })
    28. 
-
  28.   *  .on("end", function(){
    29. 
-
  29.   *      console.log("done");
    30. 
-
  30.   *  })
    31. 
-
  31.   *  .parse();
    32. 
-
  32.   * ```
    33. 
-
  33.   *
    34. 
-
  34.   * You may also parse a stream.
    35. 
-
  35.   *
    36. 
-
  36.   * ```javascript
    37. 
-
  37.   * var stream = fs.createReadStream("my.csv");
    38. 
-
  38.   *
    39. 
-
  39.   * csv(stream)
    40. 
-
  40.   *  .on("data", function(data){
    41. 
-
  41.   *      console.log(data):
    42. 
-
  42.   *  })
    43. 
-
  43.   *  .on("end", function(){
    44. 
-
  44.   *      console.log("done");
    45. 
-
  45.   *  })
    46. 
-
  46.   *  .parse();
    47. 
-
  47.   *
    48. 
-
  48.   * ```
    49. 
-
  49.   *
    50. 
-
  50.   * If you expect the first line your csv to headers you may pass a headers option in. Setting the headers option will
    51. 
-
  51.   * cause change each row to an object rather than an array.
    52. 
-
  52.   *
    53. 
-
  53.   * ```javascript
    54. 
-
  54.   * var stream = fs.createReadStream("my.csv");
    55. 
-
  55.   *
    56. 
-
  56.   * csv(stream, {headers : true})
    57. 
-
  57.   *  .on("data", function(data){
    58. 
-
  58.   *      console.log(data):
    59. 
-
  59.   *  })
    60. 
-
  60.   *  .on("end", function(){
    61. 
-
  61.   *      console.log("done");
    62. 
-
  62.   *  })
    63. 
-
  63.   *  .parse();
    64. 
-
  64.   *
    65. 
-
  65.   * ```
    66. 
-
  66.   *
    67. 
-
  67.   * You may alternatively pass an array of header names which must match the order of each column in the csv, otherwise
    68. 
-
  68.   * the data columns will not match.
    69. 
-
  69.   *
    70. 
-
  70.   * ```javascript
    71. 
-
  71.   * var stream = fs.createReadStream("my.csv");
    72. 
-
  72.   *
    73. 
-
  73.   * csv(stream, {headers : ["firstName", "lastName", "address"]})
    74. 
-
  74.   *  .on("data", function(data){
    75. 
-
  75.   *      console.log(data):
    76. 
-
  76.   *  })
    77. 
-
  77.   *  .on("end", function(){
    78. 
-
  78.   *      console.log("done");
    79. 
-
  79.   *  })
    80. 
-
  80.   *  .parse();
    81. 
-
  81.   *
    82. 
-
  82.   * ```
    83. 
-
  83.   *
    84. 
-
  84.   * If your data may include empty rows, the sort Excel might include at the end of the file for instance, you can ignore
    85. 
-
  85.   * these by including the `ignoreEmpty` option.
    86. 
-
  86.   *
    87. 
-
  87.   * Any rows consisting of nothing but empty strings and/or commas will be skipped, without emitting a 'data' or 'error' event.
    88. 
-
  88.   *
    89. 
-
  89.   * ```javascript
    90. 
-
  90.   * var stream = fs.createReadStream("my.csv");
    91. 
-
  91.   *
    92. 
-
  92.   * csv(stream, {ignoreEmpty: true})
    93. 
-
  93.   *  .on("data", function(data){
    94. 
-
  94.   *      console.log(data):
    95. 
-
  95.   *  })
    96. 
-
  96.   *  .on("end", function(){
    97. 
-
  97.   *      console.log("done");
    98. 
-
  98.   *  })
    99. 
-
  99.   *  .parse();
    100. 
-
  100.   *
    101. 
-
  101.   * ```
    102. 
-
  102.   *
    103. 
-
  103.   * ### Validating
    104. 
-
  104.   *
    105. 
-
  105.   * You can validate each row in the csv by providing a validate handler. If a row is invalid then a `data-invalid` event
    106. 
-
  106.   * will be emitted with the row and the index.
    107. 
-
  107.   *
    108. 
-
  108.   * ```javascript
    109. 
-
  109.   * var stream = fs.createReadStream("my.csv");
    110. 
-
  110.   *
    111. 
-
  111.   * csv(stream, {headers : true})
    112. 
-
  112.   *  .validate(function(data){
    113. 
-
  113.   *      return data.age < 50; //all persons must be under the age of 50
    114. 
-
  114.   *  })
    115. 
-
  115.   *  .on("data-invalid", function(data){
    116. 
-
  116.   *      //do something with invalid row
    117. 
-
  117.   *  })
    118. 
-
  118.   *  .on("data", function(data){
    119. 
-
  119.   *      console.log(data):
    120. 
-
  120.   *  })
    121. 
-
  121.   *  .on("end", function(){
    122. 
-
  122.   *      console.log("done");
    123. 
-
  123.   *  })
    124. 
-
  124.   *  .parse();
    125. 
-
  125.   *
    126. 
-
  126.   * ```
    127. 
-
  127.   *
    128. 
-
  128.   * ### Transforming
    129. 
-
  129.   *
    130. 
-
  130.   * You can transform data by providing in a transform function. What is returned from the transform function will
    131. 
-
  131.   * be provided to validate and emitted as a row.
    132. 
-
  132.   *
    133. 
-
  133.   * ```javascript
    134. 
-
  134.   * var stream = fs.createReadStream("my.csv");
    135. 
-
  135.   *
    136. 
-
  136.   * csv(stream)
    137. 
-
  137.   *  .transform(function(data){
    138. 
-
  138.   *      return data.reverse(); //reverse each row.
    139. 
-
  139.   *  })
    140. 
-
  140.   *  .on("data", function(data){
    141. 
-
  141.   *      console.log(data):
    142. 
-
  142.   *  })
    143. 
-
  143.   *  .on("end", function(){
    144. 
-
  144.   *      console.log("done");
    145. 
-
  145.   *  })
    146. 
-
  146.   *  .parse();
    147. 
-
  147.   *
    148. 
-
  148.   * ```
    149. 
-
  149.   * @footer
    150. 
-
  150.   * ## License
    151. 
-
  151.   *
    152. 
-
  152.   * MIT <https://github.com/C2FO/fast-csv/raw/master/LICENSE>
    153. 
-
  153.   *
    154. 
-
  154.   * ##Meta
    155. 
-
  155.   * * Code: `git clone git://github.com/C2FO/fast-csv.git`
    156. 
-
  156.   * * Website: <http://c2fo.com>
    157. 
-
  157.   * * Twitter: [http://twitter.com/c2fo](http://twitter.com/c2fo) - 877.465.4045
    158. 
-
  158.   */
    159. 
-
  159.  
    160. 
-
  160. 1var fs = require("fs"),
    161. 
-
  161.      _ = require("extended")().register(require("is-extended")).register(require("object-extended")),
    162. 
-
  162.      EventEmitter = require("events").EventEmitter,
    163. 
-
  163.      util = require("util"),
    164. 
-
  164.      out = process.stdout,
    165. 
-
  165.      Stream = require("stream").Stream;
    166. 
-
  166.  
    167. 
-
  167.  
    168. 
-
  168. 1var VALIDATE = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
    169. 
-
  169. 1var EMPTY = /^\s*(?:''|"")?\s*(?:,\s*(?:''|"")?\s*)*$/;
    170. 
-
  170. 1var VALUE = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
    171. 
-
  171. 1var LINE_SPLIT = /[\r\n|\r|\n]/;
    172. 
-
  172.  
    173. 
-
  173. 1function Parser(options) {
    174. 
-
  174. 14    EventEmitter.call(this);
    175. 
-
  175. 14    this._parsedHeaders = false;
    176. 
-
  176. 14    this._rowCount = 0;
    177. 
-
  177. 14    options = options || {};
    178. 
-
  178. 14    this._headers = options.headers;
    179. 
-
  179. 14    this._ignoreEmpty = options.ignoreEmpty;
    180. 
-
  180.  }
    181. 
-
  181. 1util.inherits(Parser, EventEmitter);
    182. 
-
  182.  
    183. 
-
  183. 1_(Parser).extend({
    184. 
-
  184.      __parseLine: function __parseLineData(data, index, ignore) {
    185. 
-
  185.          // Return NULL if input string is not well formed CSV string.
    186. 
-
  186.          //regexp based on http://stackoverflow.com/a/8497474
    187. 
-
  187. 108        var ignoreEmpty = this._ignoreEmpty;
    188. 
-
  188. 108        if (!VALIDATE.test(data)) {
    189. 
-
  189. 1            this.emit("error", new Error("Invalid row " + data));
    190. 
-
  190. 1            return null;
    191. 
-
  191. 107        } else if (_.isBoolean(ignoreEmpty) && ignoreEmpty && EMPTY.test(data)) {
    192. 
-
  192. 5            return null;
    193. 
-
  193.          }
    194. 
-
  194. 102        var a = [];
    195. 
-
  195. 102        data.replace(VALUE, function lineReplace(m0, m1, m2, m3) {
    196. 
-
  196. 353            var item;
    197. 
-
  197. 353            if (m1 !== undefined) {
    198. 
-
  198. 9                item = m1.replace(/\\'/g, "'");
    199. 
-
  199. 344            } else if (m2 !== undefined) {
    200. 
-
  200. 35                item = m2.replace(/\\"/g, '"');
    201. 
-
  201. 309            } else if (m3 !== undefined) {
    202. 
-
  202. 309                item = m3;
    203. 
-
  203.              }
    204. 
-
  204. 353            a.push(item);
    205. 
-
  205. 353            return ''; // Return empty string.
    206. 
-
  206.          }.bind(this));
    207. 
-
  207.          // Handle special case of empty last value.
    208. 
-
  208. 102        if (/,\s*$/.test(data)) {
    209. 
-
  209. 10            a.push('');
    210. 
-
  210.          }
    211. 
-
  211. 102        if (!ignore) {
    212. 
-
  212. 93            a = this._transform(a, index);
    213. 
-
  213. 93            if (this._validate(a, index)) {
    214. 
-
  214. 89                return a;
    215. 
-
  215.              } else {
    216. 
-
  216. 4                this.emit("data-invalid", a, index);
    217. 
-
  217.              }
    218. 
-
  218.          } else {
    219. 
-
  219. 9            return a;
    220. 
-
  220.          }
    221. 
-
  221.      },
    222. 
-
  222.  
    223. 
-
  223.      _parse: function _parseLine(data) {
    224. 
-
  224. 22        var row, parseLine = this.__parseLine.bind(this), emitRow = this.emit.bind(this, "data"), count = 0;
    225. 
-
  225. 22        if (!this._parsedHeaders) {
    226. 
-
  226. 11            var headers = this._headers;
    227. 
-
  227. 11            if (_.isBoolean(headers) && headers) {
    228. 
-
  228. 9                headers = parseLine(data.shift(), 0, true);
    229. 
-
  229.              }
    230. 
-
  230. 11            if (_.isArray(headers)) {
    231. 
-
  231. 10                var headersLength = headers.length,
    232. 
-
  232.                      orig = this._transform.bind(this);
    233. 
-
  233. 10                this._transform = function (data, index) {
    234. 
-
  234. 84                    var ret = {};
    235. 
-
  235. 84                    for (var i = 0; i < headersLength; i++) {
    236. 
-
  236. 296                        ret[headers[i]] = data[i];
    237. 
-
  237.                      }
    238. 
-
  238. 84                    return orig(ret, index);
    239. 
-
  239.                  };
    240. 
-
  240.              }
    241. 
-
  241. 11            this._parsedHeaders = true;
    242. 
-
  242.          }
    243. 
-
  243. 22        for (var i = 0, l = data.length; i < l; i++) {
    244. 
-
  244. 99            row = data[i];
    245. 
-
  245. 99            if (row) {
    246. 
-
  246. 99                var dataRow = parseLine(row, count);
    247. 
-
  247. 99                if (dataRow) {
    248. 
-
  248. 89                    count = this._rowCount++;
    249. 
-
  249. 89                    emitRow(dataRow, count);
    250. 
-
  250.                  }
    251. 
-
  251.              }
    252. 
-
  252.          }
    253. 
-
  253.      },
    254. 
-
  254.      from: function _from(from) {
    255. 
-
  255. 14        this.__from = from;
    256. 
-
  256. 14        return this;
    257. 
-
  257.      },
    258. 
-
  258.      parse: function _parse(from) {
    259. 
-
  259. 12        from = from || this.__from;
    260. 
-
  260. 12        if (_.isString(from)) {
    261. 
-
  261. 10            from = fs.createReadStream(from);
    262. 
-
  262. 10            from.on("end", function () {
    263. 
-
  263. 10                from.destroy();
    264. 
-
  264.              });
    265. 
-
  265.          }
    266. 
-
  266. 12        if (_.isObject(from) && from instanceof Stream) {
    267. 
-
  267. 11            var lines = "", parse = this._parse.bind(this), end = this.emit.bind(this, "end");
    268. 
-
  268. 11            from.on("data", function streamOnData(data) {
    269. 
-
  269. 11                var lineData = (lines + data).trim().split(LINE_SPLIT);
    270. 
-
  270. 11                if (lineData.length > 1) {
    271. 
-
  271. 11                    lines = lineData.pop();
    272. 
-
  272. 11                    parse(lineData);
    273. 
-
  273.                  } else {
    274. 
-
  274. 0                    lines += data;
    275. 
-
  275.                  }
    276. 
-
  276.              });
    277. 
-
  277. 11            from.on("end", function streamOnEnd() {
    278. 
-
  278. 11                parse(lines.split(LINE_SPLIT));
    279. 
-
  279. 11                end(this._rowCount);
    280. 
-
  280.              }.bind(this));
    281. 
-
  281.          } else {
    282. 
-
  282. 1            throw new TypeError("fast-csv.Parser#parse from must be a path or ReadableStream");
    283. 
-
  283.          }
    284. 
-
  284. 11        return this;
    285. 
-
  285.      },
    286. 
-
  286.  
    287. 
-
  287.      _validate: function (data, index) {
    288. 
-
  288. 84        return true;
    289. 
-
  289.      },
    290. 
-
  290.      _transform: function (data, index) {
    291. 
-
  291. 84        return data;
    292. 
-
  292.      },
    293. 
-
  293.      validate: function (cb) {
    294. 
-
  294. 2        if (!_.isFunction(cb)) {
    295. 
-
  295. 1            throw new TypeError("fast-csv.Parser#validate requires a function");
    296. 
-
  296.          }
    297. 
-
  297. 1        this._validate = cb;
    298. 
-
  298. 1        return this;
    299. 
-
  299.      },
    300. 
-
  300.      transform: function (cb) {
    301. 
-
  301. 2        if (!_.isFunction(cb)) {
    302. 
-
  302. 1            throw new TypeError("fast-csv.Parser#transform requires a function");
    303. 
-
  303.          }
    304. 
-
  304. 1        this._transform = cb;
    305. 
-
  305. 1        return this;
    306. 
-
  306.      }
    307. 
-
  307.  });
    308. 
-
  308.  
    309. 
-
  309.  /**
    310. 
-
  310.   * Entry point to `fast-csv`. `fast-csv` does not store rows its proccesses each row and emits it. If you wish to save
    311. 
-
  311.   * every row into an array you must store them yourself by using the `data` event. Once all rows are done processing
    312. 
-
  312.   * the `end` event is emitted.
    313. 
-
  313.   *
    314. 
-
  314.   * Invoke to parse a csv file.
    315. 
-
  315.   *
    316. 
-
  316.   * @name fast-csv
    317. 
-
  317.   * @param location
    318. 
-
  318.   * @param options
    319. 
-
  319.   * @return {*}
    320. 
-
  320.   */
    321. 
-
  321. 1module.exports = function parse(location, options) {
    322. 
-
  322. 14    return new Parser(options).from(location);
    323. 
-
  323.  };
-
- - - - - \ No newline at end of file diff --git a/docs/History.html b/docs/History.html index 3d023426..b5913a3d 100644 --- a/docs/History.html +++ b/docs/History.html @@ -176,6 +176,14 @@ +

v0.6.0

+

v0.5.7