feat: xlsx import

Chia-Network · Jan 25, 2022 · ed77312 · ed77312
1 parent 34a688c
commit ed77312
Show file tree

Hide file tree

Showing 5 changed files with 124 additions and 9 deletions.
diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js
@@ -1,5 +1,5 @@
 import _ from 'lodash';
-
+import xlsx from 'node-xlsx';
 import { uuid as uuidv4 } from 'uuidv4';
 
 import {
@@ -10,7 +10,7 @@ import {
   Vintage,
   CoBenefit,
   RelatedProject,
-  Organization,
+  Organization, Unit,
 } from '../models';
 
 import {
@@ -26,7 +26,7 @@ import {
 } from '../utils/data-assertions';
 
 import { createProjectRecordsFromCsv } from '../utils/csv-utils';
-import { createXlsFromSequelizeResults, sendXls } from '../utils/xls';
+import {tableDataFromXlsx, createXlsFromSequelizeResults, sendXls, updateTablesWithData} from '../utils/xls';
 import * as stream from "stream";
 
 export const create = async (req, res) => {
@@ -68,7 +68,6 @@ export const create = async (req, res) => {
 export const findAll = async (req, res) => {
   let { page, limit, search, orgUid, columns, xls } = req.query;
   let where = orgUid ? { orgUid } : undefined;
-
   const includes = Project.getAssociatedModels();
 
   if (columns) {
@@ -143,6 +142,24 @@ export const findOne = async (req, res) => {
   res.json(await Project.findOne(query));
 };
 
+export const updateFromXLS = async (req, res) => {
+  const { files } = req;
+
+  if(files && files.xlsx) {
+    const xlsxParsed = xlsx.parse(files.xlsx.data);
+    const stagedDataItems = tableDataFromXlsx(xlsxParsed, Project);
+    await updateTablesWithData(stagedDataItems);
+    res.json({
+      message: 'Updates from xlsx added to staging',
+    });
+  } else {
+    res.status(400).json({
+      message: 'File not received',
+      error: 'File not received',
+    });
+  }
+}
+
 export const update = async (req, res) => {
   try {
     const originalRecord = await assertProjectRecordExists(

diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js
@@ -3,7 +3,7 @@
 import _ from 'lodash';
 import { uuid as uuidv4 } from 'uuidv4';
 
-import { Staging, Unit, Qualification, Vintage, Organization } from '../models';
+import {Staging, Unit, Qualification, Vintage, Organization, Project} from '../models';
 
 import {
   columnsToInclude,
@@ -21,7 +21,8 @@ import {
 } from '../utils/data-assertions';
 
 import { createUnitRecordsFromCsv } from '../utils/csv-utils';
-import { createXlsFromSequelizeResults, sendXls } from "../utils/xls";
+import {createXlsFromSequelizeResults, sendXls, tableDataFromXlsx, updateTablesWithData} from "../utils/xls";
+import xlsx from "node-xlsx";
 
 export const create = async (req, res) => {
   try {
@@ -160,6 +161,24 @@ export const findOne = async (req, res) => {
   );
 };
 
+export const updateFromXLS = async (req, res) => {
+  const { files } = req;
+
+  if(files && files.xlsx) {
+    const xlsxParsed = xlsx.parse(files.xlsx.data);
+    const stagedDataItems = tableDataFromXlsx(xlsxParsed, Unit);
+    await updateTablesWithData(stagedDataItems);
+    res.json({
+      message: 'Updates from xlsx added to staging',
+    });
+  } else {
+    res.status(400).json({
+      message: 'File not received',
+      error: 'File not received',
+    });
+  }
+}
+
 export const update = async (req, res) => {
   try {
     const originalRecord = await assertUnitRecordExists(

diff --git a/src/routes/v1/resources/projects.js b/src/routes/v1/resources/projects.js
@@ -32,6 +32,11 @@ ProjectRouter.put(
   ProjectController.update,
 );
 
+ProjectRouter.put(
+  '/xlsx',
+  ProjectController.updateFromXLS,
+);
+
 ProjectRouter.delete(
   '/',
   validator.body(projectsDeleteSchema),

diff --git a/src/routes/v1/resources/units.js b/src/routes/v1/resources/units.js
@@ -38,4 +38,9 @@ UnitRouter.post(
 
 UnitRouter.post('/batch', UnitController.batchUpload);
 
+UnitRouter.put(
+  '/xlsx',
+  UnitController.updateFromXLS,
+);
+
 export { UnitRouter };
diff --git a/src/utils/xls.js b/src/utils/xls.js
@@ -1,9 +1,28 @@
-import {sequelize} from "../models/database.js";
-import {Project} from "../models/index.js";
+import {
+  Project,
+  CoBenefit,
+  ProjectLocation,
+  Qualification,
+  Rating,
+  RelatedProject,
+  Unit,
+  Vintage,
+  Staging
+} from './../models';
 
 import xlsx from 'node-xlsx';
 import stream from "stream";
 
+const associations = (model) => model.getAssociatedModels().map(model => {
+  if (typeof model === 'object') {
+    return model.model;
+  } else {
+    return model;
+  }
+});
+
+const capitalize = ([firstLetter, ...restOfWord]) => firstLetter.toUpperCase() + restOfWord.join('');
+
 export const sendXls = (name, bytes, response) => {
   const readStream = new stream.PassThrough();
   readStream.end(bytes);
@@ -114,8 +133,58 @@ export const createXlsFromSequelizeResults = (rows, model, hex = false, csv = fa
   if (!csv) {
     return xlsx.build(Object.values(xlsData));
   } else {
-    return Object.values(xlsData).map(({data}) => data);
+    return xlsData;
   }
 
 }
 
+export const tableDataFromXlsx = (xlsx, model) => {
+  return xlsx.reduce((stagingData, { data, name }) => {
+    const dataModel = [...associations(model), model].find((m) => {
+      const modelName = name.slice(0, -1);
+      const assocModelName = modelName.split('_');
+      if (assocModelName.length > 1) {
+        assocModelName[1] = capitalize(assocModelName[1]);
+      }
+      return m.name === name.slice(0, -1) || m.name === assocModelName.join('');
+    });
+    if (dataModel) {
+      const columnNames = data.shift();
+      for (const [_i, dataRow] of data.entries()) {
+        if (!Object.keys(stagingData).includes(dataModel.name)) {
+          stagingData[dataModel.name] = {model: dataModel, data: []};
+        }
+        const row = {}
+        for (let [columnIndex, columnData] of dataRow.entries()) {
+          if (columnData === 'null') {
+            columnData = null;
+          }
+          row[columnNames[columnIndex]] = columnData;
+        }
+        stagingData[dataModel.name].data.push(row)
+      }
+    }
+    return stagingData;
+  }, {});
+}
+
+export const updateTablesWithData = async (tableData) => {
+  const allStaged = [];
+
+  for (let [_i, {model, data}] of Object.values(tableData).entries()) {
+    for (let row of data) {
+      const exists = Object.keys(row).includes(model.primaryKeyAttributes[0]) &&
+        row[model.primaryKeyAttributes[0]].length &&
+        Boolean(await model.findByPk(row[model.primaryKeyAttributes[0]]));
+
+      allStaged.push({
+        uuid: data[model.primaryKeyAttributes[0]],
+        action: exists ? 'UPDATE' : 'INSERT',
+        table: model.tableName,
+        row,
+      });
+    }
+  }
+
+  await Staging.bulkCreate(allStaged);
+}