From 52965eb5b41a7005e5d0afe7ae4e4fce6c4544d7 Mon Sep 17 00:00:00 2001 From: Mike Keen Date: Sun, 30 Jan 2022 03:34:47 -0500 Subject: [PATCH] feat: finalize import/export --- src/controllers/project.controller.js | 4 +- src/controllers/units.controller.js | 5 ++- src/models/projects/projects.model.js | 2 + src/models/units/units.model.js | 1 + src/models/units/units.stub.json | 2 + src/utils/xls.js | 60 +++++++++++++++++++++++---- 6 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js index 5e4a671b..3feb0e72 100644 --- a/src/controllers/project.controller.js +++ b/src/controllers/project.controller.js @@ -33,7 +33,7 @@ import { tableDataFromXlsx, createXlsFromSequelizeResults, sendXls, - updateTablesWithData, + updateTableWithData, collapseTablesData, } from '../utils/xls'; export const create = async (req, res) => { @@ -190,7 +190,7 @@ export const updateFromXLS = async (req, res) => { const xlsxParsed = xlsx.parse(files.xlsx.data); const stagedDataItems = tableDataFromXlsx(xlsxParsed, Project); - await updateTablesWithData(stagedDataItems); + await updateTableWithData(collapseTablesData(stagedDataItems, Project), Project); res.json({ message: 'Updates from xlsx added to staging', diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js index 23dc8df5..6f21ac95 100644 --- a/src/controllers/units.controller.js +++ b/src/controllers/units.controller.js @@ -22,10 +22,11 @@ import { import { createUnitRecordsFromCsv } from '../utils/csv-utils'; import { + collapseTablesData, createXlsFromSequelizeResults, sendXls, tableDataFromXlsx, - updateTablesWithData, + updateTableWithData, } from '../utils/xls'; import xlsx from 'node-xlsx'; @@ -196,7 +197,7 @@ export const updateFromXLS = async (req, res) => { const xlsxParsed = xlsx.parse(files.xlsx.data); const stagedDataItems = tableDataFromXlsx(xlsxParsed, Unit); - await updateTablesWithData(stagedDataItems); + await updateTableWithData(collapseTablesData(stagedDataItems, Unit), Unit); res.json({ message: 'Updates from xlsx added to staging', diff --git a/src/models/projects/projects.model.js b/src/models/projects/projects.model.js index d0ad6e38..1f4e2ee3 100644 --- a/src/models/projects/projects.model.js +++ b/src/models/projects/projects.model.js @@ -36,6 +36,8 @@ class Project extends Model { static changes = new rxjs.Subject(); static defaultColumns = Object.keys(ModelTypes); static validateImport = projectsUpdateSchema + static virtualFieldList = {}; + static getAssociatedModels = () => [ ProjectLocation, Label, diff --git a/src/models/units/units.model.js b/src/models/units/units.model.js index d8397a4b..6b7f5c22 100644 --- a/src/models/units/units.model.js +++ b/src/models/units/units.model.js @@ -60,6 +60,7 @@ class Unit extends Model { static stagingTableName = 'Units'; static changes = new rxjs.Subject(); static validateImport = unitsUpdateSchema; + static virtualFieldList = virtualFields; static defaultColumns = Object.keys( Object.assign({}, ModelTypes, virtualFields), diff --git a/src/models/units/units.stub.json b/src/models/units/units.stub.json index 0fca568b..acec0d40 100644 --- a/src/models/units/units.stub.json +++ b/src/models/units/units.stub.json @@ -1,6 +1,8 @@ [ { "warehouseUnitId": "5c960ac1-a180-45a4-9850-be177e26d2fb", + "projectLocationId": "89adcb9a-dc10-41aa-8606-c1ce1545dcbd", + "vintageYear": 2020, "orgUid": "f1c54511-865e-4611-976c-7c3c1f704662", "unitOwner": "f1c54511-865e-4611-976c-7c3c1f704662", "countryJurisdictionOfOwner": "USA", diff --git a/src/utils/xls.js b/src/utils/xls.js index e9cf7923..02f7c17d 100644 --- a/src/utils/xls.js +++ b/src/utils/xls.js @@ -77,7 +77,9 @@ export const createXlsFromSequelizeResults = ( const initialReduceValue = {}; initialReduceValue[model.name] = { name: model.name + 's', - data: [columnsInMainSheet], + data: [columnsInMainSheet.filter(colName => { + return !(excludeOrgUid && colName === 'orgUid'); + })], }; const xlsData = rows.reduce((sheets, row) => { @@ -97,13 +99,17 @@ export const createXlsFromSequelizeResults = ( sheets[mainColName + 's'] = { name: mainColName + 's', data: [ - Object.keys(row[mainColName].filter(colName => { - return !(excludeOrgUid && colName === 'orgUid'); - })).concat([ + Object.keys(row[mainColName]).concat([ model.name.split('_').join('') + 'Id', ]), ], - }; + } + + console.log(Object.keys(row[mainColName]).filter(colName => { + return !(excludeOrgUid && colName === 'orgUid'); + }).concat([ + model.name.split('_').join('') + 'Id', + ])) } sheets[mainColName + 's'].data.push( Object.values(row[mainColName]) @@ -116,7 +122,9 @@ export const createXlsFromSequelizeResults = ( if (row[mainColName] === null) { row[mainColName] = 'null'; } + //console.log(mainColName, "//") if (!(excludeOrgUid && mainColName === 'orgUid')) { + //console.log(mainColName, "!!!"); mainXlsRow.push(encodeValue(row[mainColName], hex)); } } @@ -210,7 +218,10 @@ export const tableDataFromXlsx = (xlsx, model) => { if (columnData === 'null') { columnData = null; } - row[columnNames[columnIndex]] = columnData; + // Ignore virtual fields + if (!Object.keys(model.virtualFieldList).includes(columnNames[columnIndex])) { + row[columnNames[columnIndex]] = columnData; + } } delete row.orgUid; stagingData[dataModel.name].data.push(row); @@ -220,11 +231,42 @@ export const tableDataFromXlsx = (xlsx, model) => { }, {}); }; -export const updateTablesWithData = async (tableData) => { +export const collapseTablesData = (tableData, model) => { + const collapsed = {[model.name]: tableData[model.name]}; + + let associations = model.getAssociatedModels().map((model) => { + if (typeof model === 'object') { + return model.model; + } else { + return model; + } + }); + + for (const [i, data] of collapsed[model.name].data.entries()) { + for (const {name: association} of associations) { + if (['issuance'].includes(association)) { + collapsed[model.name].data[i][association] = tableData[association].data.find((row) => { + return row[model.name + 'Id'] === collapsed[model.name].data[i][association + 'Id']; + }) + } else { + collapsed[model.name].data[i][association + 's'] = tableData[association].data.filter((row) => { + return row[model.name + 'Id'] === collapsed[model.name].data[i][tableData[model.name].model.primaryKeyAttributes[0]]; + }); + } + } + } + + return collapsed; +} + +export const updateTableWithData = async (tableData, model) => { + if (!['project', 'unit'].includes(model.name)) { + throw 'Bulk import is only supported for projects and units'; + } // using a transaction ensures either everything is uploaded or everything fails await sequelize.transaction(async () => { const { orgUid } = await Organization.getHomeOrg(); - + for (let [, { model, data }] of Object.values(tableData).entries()) { for (let row of data) { const existingRecord = await model.findByPk( @@ -241,7 +283,7 @@ export const updateTablesWithData = async (tableData) => { row.orgUid = orgUid; } - const validation = model.validateImport.validate(row); + const validation = model.validateImport.validate(row, { stripUnknown: true }); if (!validation.error) { await Staging.upsert({