diff --git a/api-tests/earnings-api.spec.js b/api-tests/earnings-api.spec.js index 49b3e72..da7ac0f 100644 --- a/api-tests/earnings-api.spec.js +++ b/api-tests/earnings-api.spec.js @@ -504,29 +504,34 @@ describe('Earnings API tests.', () => { }); }); - // describe('Earnings BATCH GET', () => { - // const binaryParser = (res, callback) => { - // res.setEncoding('binary'); - // res.data = ''; - // res.on('data', function (chunk) { - // res.data += chunk; - // }); - // res.on('end', function () { - // callback(null, Buffer.from(res.data, 'binary')); - // }); - // }; - // it(`Should get earnings successfully`, function (done) { - // request(server) - // .get(`/earnings/batch`) - // .expect('Content-Type', 'text/csv; charset=utf-8') - // .buffer() - // .parse(binaryParser) - // .expect(200) - // .end(function (err, res) { - // if (err) return done(err); - // expect(res.body instanceof Buffer).to.be.true; - // return done(); - // }); - // }); - // }); + describe('Earnings BATCH GET', () => { + const binaryParser = (res, callback) => { + console.log('here'); + res.setEncoding('binary'); + res.data = ''; + const headers = 'earnings_id,worker_id,phone,currency,amount,status'; + let returnedHeadersEqlExpectedHeaders = false; + res.on('data', function (chunk) { + if (chunk === headers) returnedHeadersEqlExpectedHeaders = true; + res.data += chunk; + }); + res.on('end', function () { + expect(returnedHeadersEqlExpectedHeaders).to.be.true; + callback(null, Buffer.from(res.data, 'binary')); + }); + }; + it(`Should get earnings successfully`, function (done) { + request(server) + .get(`/earnings/batch`) + .expect('Content-Type', 'text/csv; charset=utf-8') + .buffer() + .parse(binaryParser) + .expect(200) + .end(function (err, res) { + if (err) return done(err); + expect(res.body instanceof Buffer).to.be.true; + return done(); + }); + }); + }); }); diff --git a/package-lock.json b/package-lock.json index 1cf248a..7bf135f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.1.5", "license": "GPL-3.0-or-later", "dependencies": { + "@fast-csv/format": "^4.3.5", "@sentry/node": "^5.1.0", "aws-sdk": "^2.1004.0", "body-parser": "^1.18.2", @@ -19,7 +20,6 @@ "express-async-handler": "^1.1.4", "express-validator": "^6.4.0", "joi": "^17.4.2", - "json2csv": "^5.0.6", "knex": "^0.21.5", "loglevel": "^1.6.8", "multer": "^1.4.3", @@ -509,6 +509,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.17.29", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.29.tgz", + "integrity": "sha512-sd4CHI9eTJXTH2vF3RGtGkqvWRwhsSSUFsXD4oG38GZzSZ0tNPbWikd2AbOAcKxCXhOg57fL8FPxjpfSzb2pIQ==" + }, "node_modules/@hapi/hoek": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.2.1.tgz", @@ -5059,23 +5077,6 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, - "node_modules/json2csv": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.6.tgz", - "integrity": "sha512-0/4Lv6IenJV0qj2oBdgPIAmFiKKnh8qh7bmLFJ+/ZZHLjSeiL3fKKGX3UryvKPbxFbhV+JcYo9KUC19GJ/Z/4A==", - "dependencies": { - "commander": "^6.1.0", - "jsonparse": "^1.3.1", - "lodash.get": "^4.4.2" - }, - "bin": { - "json2csv": "bin/json2csv.js" - }, - "engines": { - "node": ">= 10", - "npm": ">= 6.13.0" - } - }, "node_modules/json5": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", @@ -5104,6 +5105,7 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", + "dev": true, "engines": [ "node >= 0.2.0" ] @@ -5318,10 +5320,36 @@ "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=", "dev": true }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha1-SeKM1VkBNFjIFMVHnTxmOiG/qmw=" }, "node_modules/lodash.merge": { "version": "4.6.2", @@ -9455,6 +9483,26 @@ } } }, + "@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "14.17.29", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.29.tgz", + "integrity": "sha512-sd4CHI9eTJXTH2vF3RGtGkqvWRwhsSSUFsXD4oG38GZzSZ0tNPbWikd2AbOAcKxCXhOg57fL8FPxjpfSzb2pIQ==" + } + } + }, "@hapi/hoek": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.2.1.tgz", @@ -12993,16 +13041,6 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, - "json2csv": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.6.tgz", - "integrity": "sha512-0/4Lv6IenJV0qj2oBdgPIAmFiKKnh8qh7bmLFJ+/ZZHLjSeiL3fKKGX3UryvKPbxFbhV+JcYo9KUC19GJ/Z/4A==", - "requires": { - "commander": "^6.1.0", - "jsonparse": "^1.3.1", - "lodash.get": "^4.4.2" - } - }, "json5": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", @@ -13025,7 +13063,8 @@ "jsonparse": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", + "dev": true }, "JSONStream": { "version": "1.3.5", @@ -13176,10 +13215,36 @@ "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=", "dev": true }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" + }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + }, + "lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, + "lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha1-SeKM1VkBNFjIFMVHnTxmOiG/qmw=" }, "lodash.merge": { "version": "4.6.2", diff --git a/package.json b/package.json index a08c303..b1d15f4 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "server-test": "DEBUG=express:* NODE_LOG_LEVEL=debug nodemon server/serverTest.js", "server": "nodemon server/server.js", "test-seedDB": "NODE_ENV=test mocha -r dotenv/config dotenv_config_path=.env.test --timeout 10000 --require co-mocha './**/*.spec.js'", - "test-integration-ci": "NODE_ENV=test mocha -r dotenv/config --dotenv_config_path=.env.test --exit --timeout 30000 --require co-mocha './api-tests'", + "test-integration-ci": "mocha -r dotenv/config --dotenv_config_path=.env.test --exit --timeout 30000 --require co-mocha './api-tests'", "test-watch": "NODE_ENV=test NODE_LOG_LEVEL=info mocha -r dotenv/config dotenv_config_path=.env.test --timeout 10000 --require co-mocha -w -b --ignore './server/repositories/**/*.spec.js' './server/setup.js' './server/**/*.spec.js' './__tests__/seed.spec.js' './__tests__/supertest.js'", "test-watch-debug": "NODE_ENV=test NODE_LOG_LEVEL=debug mocha -r dotenv/config dotenv_config_path=.env.test --timeout 10000 --require co-mocha -w -b --ignore './server/repositories/**/*.spec.js' './server/setup.js' './server/**/*.spec.js' './__tests__/seed.spec.js' './__tests__/supertest.js'", "prettier-fix": "prettier ./ --write", @@ -31,6 +31,7 @@ "author": "Greenstand Engineers", "license": "GPL-3.0-or-later", "dependencies": { + "@fast-csv/format": "^4.3.5", "@sentry/node": "^5.1.0", "aws-sdk": "^2.1004.0", "body-parser": "^1.18.2", @@ -41,7 +42,6 @@ "express-async-handler": "^1.1.4", "express-validator": "^6.4.0", "joi": "^17.4.2", - "json2csv": "^5.0.6", "knex": "^0.21.5", "loglevel": "^1.6.8", "multer": "^1.4.3", diff --git a/server/handlers/earningsHandler.js b/server/handlers/earningsHandler.js index 9cbcbdb..879e4d1 100644 --- a/server/handlers/earningsHandler.js +++ b/server/handlers/earningsHandler.js @@ -1,9 +1,8 @@ const Joi = require('joi'); -const { Parser, AsyncParser } = require('json2csv'); const csv = require('csvtojson'); const fs = require('fs'); const { v4: uuid } = require('uuid'); -const { Readable, Transform } = require('stream'); +const { format } = require('@fast-csv/format'); const { BatchEarning } = require('../models/Earnings'); const { uploadCsv } = require('../services/aws'); @@ -78,52 +77,26 @@ const earningsBatchGet = async (req, res, next) => { const session = new Session(); const earningsRepo = new EarningsRepository(session); - const executeGetBatchEarnings = getBatchEarnings(earningsRepo); - const { earningsStream } = await executeGetBatchEarnings(req.query); - // const input = new Readable({ objectMode: true }); - // input._read = () => {}; - // earningsStream - // .on('data', (row) => { - // console.log(row); - // input.push(BatchEarning({ ...row })); - // }) - // .on('error', (error) => { - // console.log('error', error.message); - // throw new HttpError(500, error.message); - // }) - // .on('end', () => input.push(null)); - const earningTransform = new Transform({ - objectMode: true, - transform(chunk, encoding, callback) { - console.log(BatchEarning(chunk)); - this.push(BatchEarning(chunk).toString()); - callback(); - }, - }); - // const transformedReadableStream = new Readable({ - // objectMode: true, - // read(size) { - // console.log(size); - // this.push(size); - // }, - // }); - - const asyncParser = new AsyncParser({}, { objectMode: true }); - asyncParser.throughTransform(earningTransform); - const parsingProcessor = asyncParser.fromInput(earningsStream); - try { - const csv = await parsingProcessor.promise(); - // parsingProcessor.throughTransform(earningTransform); - parsingProcessor - .on('data', (chunk) => console.log(chunk)) - .on('end', () => console.log(csv)) - .on('error', (err) => console.error(err)); - console.log(csv); - // res.header('Content-Type', 'text/csv; charset=utf-8'); - // res.attachment('batchEarnings.csv'); - // res.send(csv); - // res.end(); + const executeGetBatchEarnings = getBatchEarnings(earningsRepo); + const { earningsStream } = await executeGetBatchEarnings(req.query); + const csvStream = format({ headers: true }); + + earningsStream + .on('data', async (row) => { + csvStream.write(BatchEarning({ ...row })); + }) + .on('error', (error) => { + console.log('error', error.message); + throw new HttpError(422, error.message); + }) + .on('end', () => csvStream.end()); + + res.writeHead(200, { + 'Content-Type': 'text/csv; charset=utf-8', + 'Content-Disposition': 'attachment; filename=batchEarnings.csv', + }); + csvStream.pipe(res).on('end', () => {}); } catch (err) { console.error(err); throw new HttpError(422, err.message);