From ed77312285cf332f52af35ed8aec354f38b7af14 Mon Sep 17 00:00:00 2001 From: Mike Keen Date: Mon, 24 Jan 2022 23:23:03 -0500 Subject: [PATCH] feat: xlsx import --- src/controllers/project.controller.js | 25 +++++++-- src/controllers/units.controller.js | 23 +++++++- src/routes/v1/resources/projects.js | 5 ++ src/routes/v1/resources/units.js | 5 ++ src/utils/xls.js | 75 +++++++++++++++++++++++++-- 5 files changed, 124 insertions(+), 9 deletions(-) diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js index b25b55bf..78f721f0 100644 --- a/src/controllers/project.controller.js +++ b/src/controllers/project.controller.js @@ -1,5 +1,5 @@ import _ from 'lodash'; - +import xlsx from 'node-xlsx'; import { uuid as uuidv4 } from 'uuidv4'; import { @@ -10,7 +10,7 @@ import { Vintage, CoBenefit, RelatedProject, - Organization, + Organization, Unit, } from '../models'; import { @@ -26,7 +26,7 @@ import { } from '../utils/data-assertions'; import { createProjectRecordsFromCsv } from '../utils/csv-utils'; -import { createXlsFromSequelizeResults, sendXls } from '../utils/xls'; +import {tableDataFromXlsx, createXlsFromSequelizeResults, sendXls, updateTablesWithData} from '../utils/xls'; import * as stream from "stream"; export const create = async (req, res) => { @@ -68,7 +68,6 @@ export const create = async (req, res) => { export const findAll = async (req, res) => { let { page, limit, search, orgUid, columns, xls } = req.query; let where = orgUid ? { orgUid } : undefined; - const includes = Project.getAssociatedModels(); if (columns) { @@ -143,6 +142,24 @@ export const findOne = async (req, res) => { res.json(await Project.findOne(query)); }; +export const updateFromXLS = async (req, res) => { + const { files } = req; + + if(files && files.xlsx) { + const xlsxParsed = xlsx.parse(files.xlsx.data); + const stagedDataItems = tableDataFromXlsx(xlsxParsed, Project); + await updateTablesWithData(stagedDataItems); + res.json({ + message: 'Updates from xlsx added to staging', + }); + } else { + res.status(400).json({ + message: 'File not received', + error: 'File not received', + }); + } +} + export const update = async (req, res) => { try { const originalRecord = await assertProjectRecordExists( diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js index 7608f8af..8f26e905 100644 --- a/src/controllers/units.controller.js +++ b/src/controllers/units.controller.js @@ -3,7 +3,7 @@ import _ from 'lodash'; import { uuid as uuidv4 } from 'uuidv4'; -import { Staging, Unit, Qualification, Vintage, Organization } from '../models'; +import {Staging, Unit, Qualification, Vintage, Organization, Project} from '../models'; import { columnsToInclude, @@ -21,7 +21,8 @@ import { } from '../utils/data-assertions'; import { createUnitRecordsFromCsv } from '../utils/csv-utils'; -import { createXlsFromSequelizeResults, sendXls } from "../utils/xls"; +import {createXlsFromSequelizeResults, sendXls, tableDataFromXlsx, updateTablesWithData} from "../utils/xls"; +import xlsx from "node-xlsx"; export const create = async (req, res) => { try { @@ -160,6 +161,24 @@ export const findOne = async (req, res) => { ); }; +export const updateFromXLS = async (req, res) => { + const { files } = req; + + if(files && files.xlsx) { + const xlsxParsed = xlsx.parse(files.xlsx.data); + const stagedDataItems = tableDataFromXlsx(xlsxParsed, Unit); + await updateTablesWithData(stagedDataItems); + res.json({ + message: 'Updates from xlsx added to staging', + }); + } else { + res.status(400).json({ + message: 'File not received', + error: 'File not received', + }); + } +} + export const update = async (req, res) => { try { const originalRecord = await assertUnitRecordExists( diff --git a/src/routes/v1/resources/projects.js b/src/routes/v1/resources/projects.js index bf491b19..a1768fdc 100644 --- a/src/routes/v1/resources/projects.js +++ b/src/routes/v1/resources/projects.js @@ -32,6 +32,11 @@ ProjectRouter.put( ProjectController.update, ); +ProjectRouter.put( + '/xlsx', + ProjectController.updateFromXLS, +); + ProjectRouter.delete( '/', validator.body(projectsDeleteSchema), diff --git a/src/routes/v1/resources/units.js b/src/routes/v1/resources/units.js index 5b634339..ed30655a 100644 --- a/src/routes/v1/resources/units.js +++ b/src/routes/v1/resources/units.js @@ -38,4 +38,9 @@ UnitRouter.post( UnitRouter.post('/batch', UnitController.batchUpload); +UnitRouter.put( + '/xlsx', + UnitController.updateFromXLS, +); + export { UnitRouter }; diff --git a/src/utils/xls.js b/src/utils/xls.js index 4ab8ce05..8564dc88 100644 --- a/src/utils/xls.js +++ b/src/utils/xls.js @@ -1,9 +1,28 @@ -import {sequelize} from "../models/database.js"; -import {Project} from "../models/index.js"; +import { + Project, + CoBenefit, + ProjectLocation, + Qualification, + Rating, + RelatedProject, + Unit, + Vintage, + Staging +} from './../models'; import xlsx from 'node-xlsx'; import stream from "stream"; +const associations = (model) => model.getAssociatedModels().map(model => { + if (typeof model === 'object') { + return model.model; + } else { + return model; + } +}); + +const capitalize = ([firstLetter, ...restOfWord]) => firstLetter.toUpperCase() + restOfWord.join(''); + export const sendXls = (name, bytes, response) => { const readStream = new stream.PassThrough(); readStream.end(bytes); @@ -114,8 +133,58 @@ export const createXlsFromSequelizeResults = (rows, model, hex = false, csv = fa if (!csv) { return xlsx.build(Object.values(xlsData)); } else { - return Object.values(xlsData).map(({data}) => data); + return xlsData; } } +export const tableDataFromXlsx = (xlsx, model) => { + return xlsx.reduce((stagingData, { data, name }) => { + const dataModel = [...associations(model), model].find((m) => { + const modelName = name.slice(0, -1); + const assocModelName = modelName.split('_'); + if (assocModelName.length > 1) { + assocModelName[1] = capitalize(assocModelName[1]); + } + return m.name === name.slice(0, -1) || m.name === assocModelName.join(''); + }); + if (dataModel) { + const columnNames = data.shift(); + for (const [_i, dataRow] of data.entries()) { + if (!Object.keys(stagingData).includes(dataModel.name)) { + stagingData[dataModel.name] = {model: dataModel, data: []}; + } + const row = {} + for (let [columnIndex, columnData] of dataRow.entries()) { + if (columnData === 'null') { + columnData = null; + } + row[columnNames[columnIndex]] = columnData; + } + stagingData[dataModel.name].data.push(row) + } + } + return stagingData; + }, {}); +} + +export const updateTablesWithData = async (tableData) => { + const allStaged = []; + + for (let [_i, {model, data}] of Object.values(tableData).entries()) { + for (let row of data) { + const exists = Object.keys(row).includes(model.primaryKeyAttributes[0]) && + row[model.primaryKeyAttributes[0]].length && + Boolean(await model.findByPk(row[model.primaryKeyAttributes[0]])); + + allStaged.push({ + uuid: data[model.primaryKeyAttributes[0]], + action: exists ? 'UPDATE' : 'INSERT', + table: model.tableName, + row, + }); + } + } + + await Staging.bulkCreate(allStaged); +}