Merge pull request #31 from ZJONSSON/add-tests

Add tests

.github/workflows/test.yml

@@ -11,16 +11,13 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        node-version: [18.x]
-
     steps:
       - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
+      - name: Linting with ESLint
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: 18.x
       - run: npm install
-      - run: npm run build --if-present
+      - run: npx eslint .
+      - run: npx tsc
       - run: npm test

eslint.config.mjs

@@ -0,0 +1,33 @@
+import globals from "globals";
+import pluginJs from "@eslint/js";
+
+
+export default [
+  {
+    rules:{
+      "space-before-blocks": 2,
+      "space-infix-ops": 2,
+      "object-curly-spacing": ["error", "always"],
+      "semi": 2,
+      "no-multiple-empty-lines": 2,
+      "no-multi-spaces": 2,
+      "comma-spacing": 2,
+      "prefer-const": 2,
+      "no-trailing-spaces": 2,
+      "no-var": 2,
+      "no-unused-vars": ["error", { "caughtErrors": "none" } ],
+      "indent": [
+        "error",
+        2,
+        {
+          "MemberExpression": 1,
+          "SwitchCase": 1,
+          "ignoredNodes": ["TemplateLiteral > *"]
+        }
+      ],
+    }
+  },
+  { files: ["**/*.js"], languageOptions: { sourceType: "script" } },
+  { languageOptions: { globals: globals.node } },
+  pluginJs.configs.recommended,
+];

index.js

@@ -1,23 +1,25 @@
 #! /usr/bin/env node
 const path = require('path');
 const request = require('request');
+// eslint-disable-next-line no-redeclare
 const fetch = require('node-fetch');
 const fetchCookie = require('fetch-cookie');
 const HttpsProxyAgent = require('https-proxy-agent');
 const etl = require('etl');
 const minimist = require('minimist');
 const nconf = require('nconf')
-  .file({file: process.env.ETL_CONFIG || path.resolve(process.env.HOME || process.env.USERPROFILE,'.etlconfig.json')});
+  .file({ file: process.env.ETL_CONFIG || path.resolve(process.env.HOME || process.env.USERPROFILE || '~', '.etlconfig.json') });
 
 
 const input = require('./input');
 const output = require('./output');
 
 async function main(argv) {
-    require('ts-node').register({
+  require('ts-node').register({
     transpileOnly: argv.ts_transpile === 'false' ? false : true,
     project: argv.ts_project
   });
+
   let source = argv.source;
 
   // replace proxy from config (if found)
@@ -31,42 +33,52 @@ async function main(argv) {
   argv.etl = etl;
 
   // getProxy returns a new proxy string where {{random}} has been replaced with a random number
-  argv.getProxy = () =>  argv.proxy ? argv.proxy.replace('{{random}}',String(Math.random())) : undefined;
+  argv.getProxy = () => argv.proxy ? argv.proxy.replace('{{random}}', String(Math.random())) : undefined;
 
   // inject node-fetch with proxy injection
   argv.fetch = async (url, opt) => {
     opt = opt || {};
     if (opt.proxy && !argv.proxy) throw '--proxy missing';
     opt.headers = opt.headers || {};
     opt.headers['user-agent'] = opt.headers['user-agent'] || argv.userAgent;
-    if (argv.proxy) opt = Object.assign({agent: new HttpsProxyAgent(argv.getProxy())},opt);
+    const agent = HttpsProxyAgent(argv.getProxy());
+    if (argv.proxy) opt = Object.assign({ agent }, opt);
     const fetchFn = opt.jar ? fetchCookie(fetch, opt.jar, false) : fetch;
     return fetchFn(url, opt);
-  }
+  };
 
   // Include default request / requestAsync that use proxy (if supplied) automatically
-  argv.request = d => request(Object.assign({proxy: argv.getProxy()},d));
+  argv.request = d => request(Object.assign({ proxy: argv.getProxy() }, d));
   argv.requestAsync = d => new Promise( (resolve, reject) => {
-    request(Object.assign({proxy: argv.getProxy()},d), (err, res) => err ? reject(err) : resolve(res));
+    request(Object.assign({ proxy: argv.getProxy() }, d), (err, res) => err ? reject(err) : resolve(res));
   });
-  
+
   // If source is not explicitly defined, we assume its the first argument
   if (!source) {
     source = argv?._?.[0];
     argv._ = argv?._?.slice(1);
   }
 
-  const _input = await input(source,argv);
-  if (_input) return output(_input,argv).catch(e => {
+  if (source) {
+    argv.source_dir = argv.source_dir || source.split('/').slice(1).join('/');
+    argv.source_params = source.split('/').slice(1);
+  }
+
+  const _input = await input(source, argv);
+  if (_input) return output(_input, argv).catch(e => {
     console.error(e);
     process.exit();
   });
 }
 
+function cli(argv) {
+  return main(minimist(argv));
+}
+
+//@ts-ignore
 if (!module.parent) {
-  console.log('parents')
-  const argv = minimist(process.argv.slice(2));
-  return main(argv);
+  cli(process.argv.slice(2).concat('--exit')).catch(e => console.error(e));
 }
 
-module.exports = main;
+module.exports = main;
+module.exports.cli = cli;

input.js

@@ -1,13 +1,13 @@
 #! /usr/bin/env node
 
 const path = require('path');
-const Promise = require('bluebird');
+const Bluebird = require('bluebird');
 const nconf = require('nconf');
 const fs = require('fs');
 const { safeRequire } = require('./util');
 
-module.exports = async function(source,argv) {
-  
+module.exports = async function(source, argv) {
+
   // If source
   if (source && !source.match('http') && !fs.existsSync(source)) {
     source = /^[./]/.exec(source) ? [source] : source.split('/');
@@ -21,10 +21,10 @@ module.exports = async function(source,argv) {
   }
 
   // Load custom config for the source_type or source
-  let conf = nconf.get(argv.source_type || source);
-  
-  for (let key in conf)
-    argv['source_'+key] = argv['source_'+key] || conf[key];
+  const conf = nconf.get(argv.source_type || source);
+
+  for (const key in conf)
+    argv['source_' + key] = argv['source_' + key] || conf[key];
 
   argv.source_config = conf || {
     host: argv.source_host || argv.host || 'localhost',
@@ -37,18 +37,18 @@ module.exports = async function(source,argv) {
 
   if (argv.source_query_file) {
     if (/\.(js|ts)$/.test(argv.source_query_file)) {
-      argv.source_query = await safeRequire(path.resolve('.',argv.source_query_file));
+      argv.source_query = await safeRequire(path.resolve('.', argv.source_query_file));
       if (typeof argv.source_query === 'function') {
         argv.source_query = argv.source_query(argv);
       }
     } else {
-      argv.source_query = String(fs.readFileSync(path.resolve('.',argv.source_query_file)));
+      argv.source_query = String(fs.readFileSync(path.resolve('.', argv.source_query_file)));
     }
   }
 
   // Parse query into JSON
   if (typeof argv.source_query === 'string') {
-    try {  
+    try {
       const vm = require('vm');
       argv.source_query = vm.runInNewContext(`ret = ${argv.source_query}`);
     } catch(e) {
@@ -62,33 +62,34 @@ module.exports = async function(source,argv) {
   argv.source = source;
 
   // Resolve any injections
-  for (let key in argv) {
+  for (const key in argv) {
     if (argv[key] && key.indexOf('inject_') === 0) {
-      argv[key] = module.exports(argv[key], Object.assign({}, argv, {[key]: null})).stream(argv);
+      const inject = await module.exports(argv[key], Object.assign({}, argv, { [key]: null }));
+      argv[key] = inject.stream(argv);
     }
   }
 
-  let type,obj;
+  let obj;
 
 
   // If the file is json or csv we set the correct type
   const match = /\.(json|csv|xlsx|parquet|xml)/.exec(source);
-  type = argv.source_type || (match && match[1]) || source;
+  const type = argv.source_type || (match && match[1]) || source;
 
   // Find the matching source_type and execute
-  let sourcePath = path.resolve(__dirname,'sources',`${type}.js`);
+  const sourcePath = path.resolve(__dirname, 'sources', `${type}.js`);
   if (match || fs.existsSync(sourcePath)) {
-      obj = (await safeRequire(sourcePath))(argv);
+    obj = (await safeRequire(sourcePath))(argv);
   } else {
-      obj = await safeRequire(path.resolve('.',source));
+    obj = await safeRequire(path.resolve('.', source));
   }
-  
+
   if (!obj.stream)
     obj.stream = obj;
 
   if (argv.schema) {
     if (!argv.silent) console.log(`Using schema ${argv.schema}`);
-    Object.assign(obj,require(path.resolve('./',argv.schema)));
+    Object.assign(obj, require(path.resolve('./', argv.schema)));
   }
 
   if (!argv.silent)
@@ -97,7 +98,7 @@ module.exports = async function(source,argv) {
   if (argv.count) {
     if (!obj.recordCount)
       throw 'No Recordcount available';
-    Promise.try(() => obj.recordCount(argv))
+    Bluebird.try(() => obj.recordCount(argv))
       .then(d => console.log(`Record count: ${d}`))
       .then(() => process.exit());
   } else