feat: replace fields config by a global config (#338)

BREAKING CHANGE: `--fields-config` option has been removed, use the new `--config` option for all configuration, not just fields.
zemirco · Aug 19, 2019 · d6c1c5f · d6c1c5f
1 parent 6fd6c09
commit d6c1c5f
Show file tree

Hide file tree

Showing 11 changed files with 143 additions and 124 deletions.
diff --git a/README.md b/README.md
@@ -50,17 +50,19 @@ By default, the above script will get the latest release of json2csv. You can al
 
 `json2csv` can be called from the command line if installed globally (using the `-g` flag).
 
-```sh
+```bash
 Usage: json2csv [options]
 
+
 Options:
+
   -V, --version                        output the version number
+  -c, --config <path>                  Specify a file with a valid JSON configuration.
   -i, --input <input>                  Path and name of the incoming json file. Defaults to stdin.
   -o, --output [output]                Path and name of the resulting csv file. Defaults to stdout.
   -n, --ndjson                         Treat the input as NewLine-Delimited JSON.
   -s, --no-streaming                   Process the whole JSON array in memory instead of doing it line by line.
   -f, --fields <fields>                List of fields to process. Defaults to field auto-detection.
-  -c, --fields-config <path>           File with a fields configuration as a JSON array.
   -u, --unwind <paths>                 Creates multiple rows from a single JSON document similar to MongoDB unwind.
   -B, --unwind-blank                   When unwinding, blank out instead of repeating data.
   -F, --flatten                        Flatten nested objects.
@@ -83,6 +85,8 @@ If no output `-o` is specified the result is printed to the console standard out
 If no fields `-f` or `-c` are passed the fields of the first element are used since json2csv CLI process the items one at a time. You can use the `--no-streaming` flag to load the entire JSON in memory and get all the headers. However, keep in mind that this is slower and requires much more memory.
 Use `-p` to show the result as a table in the console.
 
+Any option pass through a config file `-c` will be overriden if a specific flag is passed as well. For example, the fields option of the config will be overriden if the fields flag `-f` is used.
+
 ### CLI examples
 
 All examples use this example [input file](https://github.com/zemirco/json2csv/blob/master/test/fixtures/json/default.json).

diff --git a/bin/json2csv.js b/bin/json2csv.js
@@ -18,10 +18,10 @@ program
   .version(pkg.version)
   .option('-i, --input <input>', 'Path and name of the incoming json file. Defaults to stdin.')
   .option('-o, --output [output]', 'Path and name of the resulting csv file. Defaults to stdout.')
+  .option('-c, --config <path>', 'Specify a file with a valid JSON configuration.')
   .option('-n, --ndjson', 'Treat the input as NewLine-Delimited JSON.')
   .option('-s, --no-streaming', 'Process the whole JSON array in memory instead of doing it line by line.')
   .option('-f, --fields <fields>', 'List of fields to process. Defaults to field auto-detection.')
-  .option('-c, --fields-config <path>', 'File with a fields configuration as a JSON array.')
   .option('-u, --unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
   .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.')
   .option('-F, --flatten', 'Flatten nested objects.')
@@ -46,8 +46,10 @@ function makePathAbsolute(filePath) {
 
 const inputPath = makePathAbsolute(program.input);
 const outputPath = makePathAbsolute(program.output);
-const fieldsConfigPath = makePathAbsolute(program.fieldsConfig);
+const configPath = makePathAbsolute(program.config);
 
+if (program.fields) program.fields = program.fields.split(',');
+if (program.unwind) program.unwind = program.unwind.split(',');
 program.delimiter = program.delimiter || ',';
 program.eol = program.eol || os.EOL;
 
@@ -59,14 +61,10 @@ process.stdout.on('error', (error) => {
   }
 });
 
-function getFields() {
-  if (fieldsConfigPath) {
-    return require(fieldsConfigPath);
-  }
-
-  return program.fields
-      ? program.fields.split(',')
-      : undefined;
+function getConfigFromFile() {
+  return configPath
+    ? require(configPath)
+    : {};
 }
 
 function getInputStream() {
@@ -152,25 +150,27 @@ function processOutput(csv) {
 
 Promise.resolve()
   .then(() => {
+    const config = Object.assign({}, getConfigFromFile(), program);
+
     const opts = {
-      fields: getFields(),
-      unwind: program.unwind ? program.unwind.split(',') : [],
-      unwindBlank: program.unwindBlank,
-      flatten: program.flatten,
-      flattenSeparator: program.flattenSeparator,
-      defaultValue: program.defaultValue,
-      quote: program.quote,
-      doubleQuote: program.doubleQuote,
-      delimiter: program.delimiter,
-      eol: program.eol,
-      excelStrings: program.excelStrings,
-      header: program.header,
-      includeEmptyRows: program.includeEmptyRows,
-      withBOM: program.withBom
+      fields: config.fields,
+      unwind: config.unwind,
+      unwindBlank: config.unwindBlank,
+      flatten: config.flatten,
+      flattenSeparator: config.flattenSeparator,
+      defaultValue: config.defaultValue,
+      quote: config.quote,
+      doubleQuote: config.doubleQuote,
+      delimiter: config.delimiter,
+      eol: config.eol,
+      excelStrings: config.excelStrings,
+      header: config.header,
+      includeEmptyRows: config.includeEmptyRows,
+      withBOM: config.withBom
     };
 
-    if (!program.streaming) {
-      return getInput()
+    if (!config.streaming) {
+      return getInput(config.ndjson)
         .then(input => new JSON2CSVParser(opts).parse(input))
         .then(processOutput);
     }
@@ -179,7 +179,7 @@ Promise.resolve()
     const input = getInputStream();
     const stream = input.pipe(transform);
 
-    if (program.output) {
+    if (config.output) {
       const outputStream = fs.createWriteStream(outputPath, { encoding: 'utf8' });
       const output = stream.pipe(outputStream);
       return new Promise((resolve, reject) => {
@@ -190,7 +190,7 @@ Promise.resolve()
       });
     }
 
-    if (!program.pretty) {
+    if (!config.pretty) {
       const output = stream.pipe(process.stdout);
       return new Promise((resolve, reject) => {
         input.on('error', reject);
@@ -205,18 +205,17 @@ Promise.resolve()
       input.on('error', reject);
       stream.on('error', reject);
       let csv = '';
-      const table = new TablePrinter(program);
+      const table = new TablePrinter(config);
       stream
         .on('data', chunk => {
           csv += chunk.toString();
-          const index = csv.lastIndexOf(program.eol);
+          const index = csv.lastIndexOf(config.eol);
           let lines = csv.substring(0, index);
           csv = csv.substring(index + 1);
 
           if (lines) {
             table.push(lines);
           }
-
         })
         .on('end', () => {
           table.end(csv);
@@ -230,8 +229,8 @@ Promise.resolve()
       err = new Error('Invalid input file. (' + err.message + ')');
     } else if (outputPath && err.message.includes(outputPath)) {
       err = new Error('Invalid output file. (' + err.message + ')');
-    } else if (fieldsConfigPath && err.message.includes(fieldsConfigPath)) {
-      err = new Error('Invalid fields config file. (' + err.message + ')');
+    } else if (configPath && err.message.indexOf(configPath) !== -1) {
+      err = new Error('Invalid config file. (' + err.message + ')');
     }
     // eslint-disable-next-line no-console
     console.error(err);

diff --git a/test/CLI.js b/test/CLI.js
@@ -168,10 +168,10 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   // });
 
   testRunner.add('should error on invalid fields config file path', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields2/fieldNames.json');
+    const opts = ' --config ' + getFixturePath('/fields2/fieldNames.json');
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/default.json') + opts, (err, stdout, stderr) => {
-      t.ok(stderr.includes('Invalid fields config file.'));
+      t.ok(stderr.indexOf('Invalid config file.') !== -1);
       t.end();
     });
   });
@@ -221,7 +221,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should name columns as specified in \'fields\' property', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/fieldNames.json');
+    const opts = ' --config ' + getFixturePath('/fields/fieldNames.json');
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/default.json') + opts, (err, stdout, stderr) => {
       t.notOk(stderr);
@@ -232,7 +232,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should support nested properties selectors', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/nested.json');
+    const opts = ' --config ' + getFixturePath('/fields/nested.json');
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/nested.json') + opts, (err, stdout, stderr) => {
       t.notOk(stderr);
@@ -254,7 +254,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('field.value function should stringify results by default', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/functionStringifyByDefault.js');
+    const opts = ' --config ' + getFixturePath('/fields/functionStringifyByDefault.js');
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/functionStringifyByDefault.json') + opts, (err, stdout, stderr) => {
       t.notOk(stderr);
@@ -265,7 +265,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('field.value function should not stringify if stringify is selected to false', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/functionNoStringify.js');
+    const opts = ' --config ' + getFixturePath('/fields/functionNoStringify.js');
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/functionNoStringify.json') + opts, (err, stdout, stderr) => {
       t.notOk(stderr);
@@ -276,7 +276,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should process different combinations in fields option', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/fancyfields.js')
+    const opts = ' --config ' + getFixturePath('/fields/fancyfields.js')
       + ' --default-value NULL';
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/fancyfields.json') + opts, (err, stdout, stderr) => {
@@ -371,7 +371,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should override \'options.defaultValue\' with \'field.defaultValue\'', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/overriddenDefaultValue.json')
+    const opts = ' --config ' + getFixturePath('/fields/overriddenDefaultValue.json')
       + ' --default-value ""';
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/overriddenDefaultValue.json') + opts, (err, stdout, stderr) => {
@@ -383,7 +383,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should use \'options.defaultValue\' when no \'field.defaultValue\'', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/overriddenDefaultValue2.js')
+    const opts = ' --config ' + getFixturePath('/fields/overriddenDefaultValue2.js')
       + ' --default-value ""';
 
     child_process.exec(cli + '-i ' + getFixturePath('/json/overriddenDefaultValue.json') + opts, (err, stdout, stderr) => {
@@ -612,7 +612,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should include empty rows when options.includeEmptyRows is true, with default values', (t) => {
-    const opts = ' --fields-config ' + getFixturePath('/fields/emptyRowDefaultValues.json')
+    const opts = ' --config ' + getFixturePath('/fields/emptyRowDefaultValues.json')
       + ' --default-value NULL'
       + ' --include-empty-rows';
 

diff --git a/test/fixtures/fields/emptyRowDefaultValues.json b/test/fixtures/fields/emptyRowDefaultValues.json
@@ -1,12 +1,14 @@
-[
-  {
-    "value": "carModel"
-  },
-  {
-    "value": "price",
-    "default": 1
-  },
-  {
-    "value": "color"
-  }
-]
+{
+  "fields": [
+    {
+      "value": "carModel"
+    },
+    {
+      "value": "price",
+      "default": 1
+    },
+    {
+      "value": "color"
+    }
+  ]
+}
diff --git a/test/fixtures/fields/fancyfields.js b/test/fixtures/fields/fancyfields.js
@@ -1,16 +1,18 @@
-module.exports = [{
-  label: 'PATH1',
-  value: 'path1'
-}, {
-  label: 'PATH1+PATH2',
-  value: row => row.path1+row.path2
-}, {
-  label: 'NEST1',
-  value: 'bird.nest1'
-},
-'bird.nest2',
-{
-  label: 'nonexistent',
-  value: 'fake.path',
-  default: 'col specific default value'
-}];
+module.exports = {
+  fields: [{
+    label: 'PATH1',
+    value: 'path1'
+  }, {
+    label: 'PATH1+PATH2',
+    value: row => row.path1+row.path2
+  }, {
+    label: 'NEST1',
+    value: 'bird.nest1'
+  },
+  'bird.nest2',
+  {
+    label: 'nonexistent',
+    value: 'fake.path',
+    default: 'col specific default value'
+  }]
+};
diff --git a/test/fixtures/fields/fieldNames.json b/test/fixtures/fields/fieldNames.json
@@ -1,7 +1,9 @@
-[{
-  "label": "Car Model",
-  "value": "carModel"
-},{
-  "label": "Price USD",
-  "value": "price"
-}]
+{
+  "fields": [{
+    "label": "Car Model",
+    "value": "carModel"
+  },{
+    "label": "Price USD",
+    "value": "price"
+  }]
+}
diff --git a/test/fixtures/fields/functionNoStringify.js b/test/fixtures/fields/functionNoStringify.js
@@ -1,5 +1,7 @@
-module.exports = [{
-  label: 'Value1',
-  value: row => row.value1.toLocaleString(),
-  stringify: false
-}];
+module.exports = {
+  fields: [{
+    label: 'Value1',
+    value: row => row.value1.toLocaleString(),
+    stringify: false
+  }]
+};
diff --git a/test/fixtures/fields/functionStringifyByDefault.js b/test/fixtures/fields/functionStringifyByDefault.js
@@ -1,4 +1,6 @@
-module.exports = [{
-  label: 'Value1',
-  value: row => row.value1.toLocaleString()
-}];
+module.exports = {
+  fields: [{
+    label: 'Value1',
+    value: row => row.value1.toLocaleString()
+  }]
+};