feat: Added flattenSeparator option (#314)

* Added flattenSeparator option * Updated docstring
zemirco · Jul 31, 2018 · 5c5de9f · 5c5de9f
1 parent b2690fe
commit 5c5de9f
Show file tree

Hide file tree

Showing 6 changed files with 80 additions and 50 deletions.
diff --git a/README.md b/README.md
@@ -45,27 +45,28 @@ $ npm install json2csv --save
 
   Options:
 
-    -V, --version                       output the version number
-    -i, --input <input>                 Path and name of the incoming json file. If not provided, will read from stdin.
-    -o, --output [output]               Path and name of the resulting csv file. Defaults to stdout.
-    -n, --ndjson                        Treat the input as NewLine-Delimited JSON.
-    -s, --no-streaming                  Process the whole JSON array in memory instead of doing it line by line.
-    -f, --fields <fields>               Specify the fields to convert.
-    -c, --fields-config <path>          Specify a file with a fields configuration as a JSON array.
-    -u, --unwind <paths>                Creates multiple rows from a single JSON document similar to MongoDB unwind.
-    -B, --unwind-blank                  When unwinding, blank out instead of repeating data.
-    -F, --flatten                       Flatten nested objects
-    -v, --default-value [defaultValue]  Specify a default value other than empty string.
-    -q, --quote [value]                 Specify an alternate quote value.
-    -Q, --double-quote [value]          Specify a value to replace double quote in strings
-    -d, --delimiter [delimiter]         Specify a delimiter other than the default comma to use.
-    -e, --eol [value]                   Specify an End-of-Line value for separating rows.
-    -E, --excel-strings                 Converts string data into normalized Excel style data
-    -H, --no-header                     Disable the column name header
-    -a, --include-empty-rows            Includes empty rows in the resulting CSV output.
-    -b, --with-bom                      Includes BOM character at the beginning of the csv.
-    -p, --pretty                        Use only when printing to console. Logs output in pretty tables.
-    -h, --help                          output usage information
+    -V, --version                        output the version number
+    -i, --input <input>                  Path and name of the incoming json file. If not provided, will read from stdin.
+    -o, --output [output]                Path and name of the resulting csv file. Defaults to stdout.
+    -n, --ndjson                         Treat the input as NewLine-Delimited JSON.
+    -s, --no-streaming                   Process the whole JSON array in memory instead of doing it line by line.
+    -f, --fields <fields>                Specify the fields to convert.
+    -c, --fields-config <path>           Specify a file with a fields configuration as a JSON array.
+    -u, --unwind <paths>                 Creates multiple rows from a single JSON document similar to MongoDB unwind.
+    -B, --unwind-blank                   When unwinding, blank out instead of repeating data.
+    -F, --flatten                        Flatten nested objects.
+    -S, --flatten-separator <separator>  Flattened keys separator.
+    -v, --default-value [defaultValue]   Specify a default value other than empty string.
+    -q, --quote [value]                  Specify an alternate quote value.
+    -Q, --double-quote [value]           Specify a value to replace double quote in strings.
+    -d, --delimiter [delimiter]          Specify a delimiter other than the default comma to use.
+    -e, --eol [value]                    Specify an End-of-Line value for separating rows.
+    -E, --excel-strings                  Converts string data into normalized Excel style data.
+    -H, --no-header                      Disable the column name header.
+    -a, --include-empty-rows             Includes empty rows in the resulting CSV output.
+    -b, --with-bom                       Includes BOM character at the beginning of the csv.
+    -p, --pretty                         Use only when printing to console. Logs output in pretty tables.
+    -h, --help                           output usage information
 ```
 
 An input file `-i` and fields `-f` are required. If no output `-o` is specified the result is logged to the console.
@@ -158,6 +159,7 @@ The programatic APIs take a configuration object very equivalent to the CLI opti
 - `unwind` - Array of Strings, creates multiple rows from a single JSON document similar to MongoDB's $unwind
 - `unwindBlank` - Boolean, unwind using blank values instead of repeating data.
 - `flatten` - Boolean, flattens nested JSON using [flat]. Defaults to `false`.
+- `flattenSeparator` - String, separator to use between nested JSON keys when `flatten` option enabled. Defaults to `.` if not specified.
 - `defaultValue` - String, default value to use when missing data. Defaults to `<empty>` if not specified. (Overridden by `fields[].default`)
 - `quote` - String, quote around cell values and column names. Defaults to `"` if not specified.
 - `doubleQuote` - String, the value to replace double quote in strings. Defaults to 2x`quotes` (for example `""`) if not specified.

diff --git a/bin/json2csv.js b/bin/json2csv.js
@@ -24,14 +24,15 @@ program
   .option('-c, --fields-config <path>', 'Specify a file with a fields configuration as a JSON array.')
   .option('-u, --unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
   .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.')
-  .option('-F, --flatten', 'Flatten nested objects')
+  .option('-F, --flatten', 'Flatten nested objects.')
+  .option('-S, --flatten-separator <separator>', 'Flattened keys separator.')
   .option('-v, --default-value [defaultValue]', 'Specify a default value other than empty string.')
   .option('-q, --quote [value]', 'Specify an alternate quote value.')
-  .option('-Q, --double-quote [value]', 'Specify a value to replace double quote in strings')
+  .option('-Q, --double-quote [value]', 'Specify a value to replace double quote in strings.')
   .option('-d, --delimiter [delimiter]', 'Specify a delimiter other than the default comma to use.')
   .option('-e, --eol [value]', 'Specify an End-of-Line value for separating rows.')
-  .option('-E, --excel-strings','Converts string data into normalized Excel style data')
-  .option('-H, --no-header', 'Disable the column name header')
+  .option('-E, --excel-strings','Converts string data into normalized Excel style data.')
+  .option('-H, --no-header', 'Disable the column name header.')
   .option('-a, --include-empty-rows', 'Includes empty rows in the resulting CSV output.')
   .option('-b, --with-bom', 'Includes BOM character at the beginning of the csv.')
   .option('-p, --pretty', 'Use only when printing to console. Logs output in pretty tables.')
@@ -156,6 +157,7 @@ Promise.resolve()
       unwind: program.unwind ? program.unwind.split(',') : [],
       unwindBlank: program.unwindBlank,
       flatten: program.flatten,
+      flattenSeparator: program.flattenSeparator,
       defaultValue: program.defaultValue,
       quote: program.quote,
       doubleQuote: program.doubleQuote,

diff --git a/json2csv b/json2csv
diff --git a/lib/JSON2CSVBase.js b/lib/JSON2CSVBase.js
@@ -22,6 +22,7 @@ class JSON2CSVBase {
       ? (processedOpts.unwind ? [processedOpts.unwind] : [])
       : processedOpts.unwind
     processedOpts.delimiter = processedOpts.delimiter || ',';
+    processedOpts.flattenSeparator = processedOpts.flattenSeparator || '.';
     processedOpts.eol = processedOpts.eol || os.EOL;
     processedOpts.quote = typeof processedOpts.quote === 'string'
       ? opts.quote
@@ -63,7 +64,7 @@ class JSON2CSVBase {
       : [row];
 
     if (this.opts.flatten) {
-      return processedRow.map(this.flatten);
+      return processedRow.map(this.flatten());
     }
 
     return processedRow;
@@ -208,34 +209,37 @@ class JSON2CSVBase {
   /**
    * Performs the flattening of a data row recursively
    *
-   * @param {Object} dataRow Original JSON object
-   * @returns {Object} Flattened object
+   * @returns {Function} Function that receives dataRow as input and outputs flattened object 
    */
-  flatten(dataRow) {
-    function step (obj, flatDataRow, currentPath) {
-      Object.keys(obj).forEach((key) => {
-        const value = obj[key];
-
-        const newPath = currentPath
-          ? `${currentPath}.${key}`
-          : key;
-
-        if (typeof value !== 'object'
-          || value === null
-          || Array.isArray(value)
-          || Object.prototype.toString.call(value.toJSON) === '[object Function]'
-          || !Object.keys(value).length) {
-          flatDataRow[newPath] = value;
-          return;
-        }
+  flatten() {
+    return (dataRow) => {
+      const separator = this.opts.flattenSeparator;
+
+      function step (obj, flatDataRow, currentPath) {
+        Object.keys(obj).forEach((key) => {
+          const value = obj[key];
+
+          const newPath = currentPath
+            ? `${currentPath}${separator}${key}`
+            : key;
+
+          if (typeof value !== 'object'
+            || value === null
+            || Array.isArray(value)
+            || Object.prototype.toString.call(value.toJSON) === '[object Function]'
+            || !Object.keys(value).length) {
+            flatDataRow[newPath] = value;
+            return;
+          }
 
-        step(value, flatDataRow, newPath);
-      });
+          step(value, flatDataRow, newPath);
+        });
 
-      return flatDataRow;
-    }
+        return flatDataRow;
+      }
 
-    return step(dataRow, {});
+      return step(dataRow, {});
+    }
   }
 
   /**

diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js
@@ -453,6 +453,25 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
       .on('error', err => t.notOk(true, err.message));
   });
 
+  testRunner.add('should support custom flatten separator', (t) => {
+    const opts = {
+      flatten: true,
+      flattenSeparator: '__',
+    };
+
+    const transform = new Json2csvTransform(opts);
+    const processor = jsonFixtures.deepJSON().pipe(transform);
+
+    let csv = '';
+    processor
+      .on('data', chunk => (csv += chunk.toString()))
+      .on('end', () => {
+        t.equal(csv, csvFixtures.flattenedCustomSeparatorDeepJSON);
+        t.end();
+      })
+      .on('error', err => t.notOk(true, err.message));
+  });
+
   testRunner.add('should unwind and flatten an object in the right order', (t) => {
     const opts = {
       unwind: ['items'],

diff --git a/test/fixtures/csv/flattenedCustomSeparatorDeepJSON.csv b/test/fixtures/csv/flattenedCustomSeparatorDeepJSON.csv
@@ -0,0 +1,2 @@
+"field1__embeddedField1","field1__embeddedField2"
+"embeddedValue1","embeddedValue2"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		"field1__embeddedField1","field1__embeddedField2"
		"embeddedValue1","embeddedValue2"