From 2a158e8dff9a0390dd19ed6c4dd782a67e6fdc97 Mon Sep 17 00:00:00 2001 From: Mike Keen Date: Sun, 30 Jan 2022 15:49:47 -0500 Subject: [PATCH] feat: finalize data import --- src/models/issuances/issuances.stub.json | 1 + src/models/labels/labels.stub.json | 3 +- src/models/units/units.model.js | 6 +- src/utils/xls.js | 109 ++++++++++++++++------- src/validations/labelUnit.validations.js | 12 +++ src/validations/labels.validations.js | 2 + 6 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 src/validations/labelUnit.validations.js diff --git a/src/models/issuances/issuances.stub.json b/src/models/issuances/issuances.stub.json index a4f33f4a..fc55f034 100644 --- a/src/models/issuances/issuances.stub.json +++ b/src/models/issuances/issuances.stub.json @@ -7,6 +7,7 @@ "endDate": "2029-03-12 00:05:45.701 +00:00", "verificationApproach": "TEST", "verificationReportDate": "2022-01-18 00:05:45.701 +00:00", + "verificationDate": "2022-01-18 00:05:45.701 +00:00", "verificationBody": "This is verified", "createdAt": "2022-01-18 00:05:45.701 +00:00", "updatedAt": "2022-01-18 00:05:45.701 +00:00" diff --git a/src/models/labels/labels.stub.json b/src/models/labels/labels.stub.json index 45af024f..5553c2bd 100644 --- a/src/models/labels/labels.stub.json +++ b/src/models/labels/labels.stub.json @@ -11,6 +11,7 @@ "unitQuantity": 5, "labelLink": "https://label.link/1", "createdAt": "2022-01-18 00:05:45.701 +00:00", - "updatedAt": "2022-01-18 00:05:45.701 +00:00" + "updatedAt": "2022-01-18 00:05:45.701 +00:00", + "labelType": "stub" } ] diff --git a/src/models/units/units.model.js b/src/models/units/units.model.js index 6b7f5c22..c819561b 100644 --- a/src/models/units/units.model.js +++ b/src/models/units/units.model.js @@ -16,7 +16,7 @@ import { createXlsFromSequelizeResults, transformFullXslsToChangeList, } from '../../utils/xls'; -import {unitsUpdateSchema} from "../../validations/index.js"; +import { unitsUpdateSchema } from '../../validations/index.js'; const { Model } = Sequelize; @@ -61,7 +61,7 @@ class Unit extends Model { static changes = new rxjs.Subject(); static validateImport = unitsUpdateSchema; static virtualFieldList = virtualFields; - + static defaultColumns = Object.keys( Object.assign({}, ModelTypes, virtualFields), ); @@ -297,8 +297,6 @@ class Unit extends Model { true, ); - console.log(insertXslsSheets); - const updateXslsSheets = createXlsFromSequelizeResults( updateRecords, Unit, diff --git a/src/utils/xls.js b/src/utils/xls.js index 57363e94..7e74097c 100644 --- a/src/utils/xls.js +++ b/src/utils/xls.js @@ -3,7 +3,7 @@ import _ from 'lodash'; import xlsx from 'node-xlsx'; import stream from 'stream'; -import { Staging, Organization } from './../models'; +import { Staging, Organization, LabelUnit } from './../models'; import { sequelize } from '../models/database'; import { assertOrgIsHomeOrg } from '../utils/data-assertions'; @@ -45,6 +45,7 @@ export const encodeValue = (value, hex = false) => { }; export const createXlsFromSequelizeResults = ( + // todo recursion rows, model, hex = false, @@ -192,8 +193,9 @@ export const createXlsFromSequelizeResults = ( }; export const tableDataFromXlsx = (xlsx, model) => { + // Todo recursion return xlsx.reduce((stagingData, { data, name }) => { - const dataModel = [...associations(model), model].find((m) => { + let dataModel = [...associations(model), model].find((m) => { const modelName = name.slice(0, -1); const assocModelName = modelName.split('_'); if (assocModelName.length > 1) { @@ -201,35 +203,40 @@ export const tableDataFromXlsx = (xlsx, model) => { } return m.name === name.slice(0, -1) || m.name === assocModelName.join(''); }); - if (dataModel) { - const columnNames = data.shift(); - for (const [, dataRow] of data.entries()) { - if (!Object.keys(stagingData).includes(dataModel.name)) { - stagingData[dataModel.name] = { model: dataModel, data: [] }; + + if (model.name === 'unit' && dataModel === undefined) { + // todo clean this up + dataModel = LabelUnit; + } + + const columnNames = data.shift(); + for (const [, dataRow] of data.entries()) { + if (!Object.keys(stagingData).includes(dataModel.name)) { + stagingData[dataModel.name] = { model: dataModel, data: [] }; + } + const row = {}; + for (let [columnIndex, columnData] of dataRow.entries()) { + if (columnData === 'null') { + columnData = null; } - const row = {}; - for (let [columnIndex, columnData] of dataRow.entries()) { - if (columnData === 'null') { - columnData = null; - } - // Ignore virtual fields - if ( - !Object.keys(model.virtualFieldList).includes( - columnNames[columnIndex], - ) - ) { - row[columnNames[columnIndex]] = columnData; - } + // Ignore virtual fields + if ( + !Object.keys(model.virtualFieldList).includes( + columnNames[columnIndex], + ) + ) { + row[columnNames[columnIndex]] = columnData; } - delete row.orgUid; - stagingData[dataModel.name].data.push(row); } + delete row.orgUid; + stagingData[dataModel.name].data.push(row); } return stagingData; }, {}); }; export const collapseTablesData = (tableData, model) => { + // Todo recursion const collapsed = { [model.name]: tableData[model.name] }; let associations = model.getAssociatedModels().map((model) => { @@ -243,24 +250,59 @@ export const collapseTablesData = (tableData, model) => { for (const [i] of collapsed[model.name].data.entries()) { for (const { name: association } of associations) { if (['issuance'].includes(association)) { + // Todo: make generic collapsed[model.name].data[i][association] = tableData[ association ].data.find((row) => { - return ( + let found = false; + + if ( row[model.name + 'Id'] === collapsed[model.name].data[i][association + 'Id'] - ); + ) { + found = true; + delete row[model.name + 'Id']; + } + return found; }); } else { collapsed[model.name].data[i][association + 's'] = tableData[ association ].data.filter((row) => { - return ( + let found = false; + if ( row[model.name + 'Id'] === - collapsed[model.name].data[i][ + tableData[model.name].data[i][ tableData[model.name].model.primaryKeyAttributes[0] ] - ); + ) { + delete row[model.name + 'Id']; + found = true; + } + return found; + }); + } + } + } + + for (const [i] of collapsed[model.name].data.entries()) { + for (const { name: association } of associations) { + if (['label'].includes(association)) { + // Todo: make generic + const tData = tableData['label_unit'].data.find((row) => { + return tableData[model.name].data[i].labels + .map((l) => l.id) + .includes(row['labelunitId']); + }); + + collapsed[model.name].data[i][association + 's'][0]['label_unit'] = + tData; + + collapsed[model.name].data[i].labels = collapsed[model.name].data[ + i + ].labels.map((l) => { + delete l.label_unit.labelunitId; + return l; }); } } @@ -271,7 +313,7 @@ export const collapseTablesData = (tableData, model) => { export const updateTableWithData = async (tableData, model) => { if (!['project', 'unit'].includes(model.name)) { - throw 'Bulk import is only supported for projects and units'; + throw 'Bulk import is only supported for projects and units'; // Technically, updateTableWithData can support any model } // using a transaction ensures either everything is uploaded or everything fails await sequelize.transaction(async () => { @@ -285,6 +327,13 @@ export const updateTableWithData = async (tableData, model) => { const exists = Boolean(existingRecord); + // Stripping out issuanceId if its included. Need to take another look at this + if (model.name === 'unit') { + delete row['issuanceId']; + } + + const validation = model.validateImport.validate(row); + if (exists) { // Assert the original record is a record your allowed to modify await assertOrgIsHomeOrg(existingRecord.orgUid); @@ -293,10 +342,6 @@ export const updateTableWithData = async (tableData, model) => { row.orgUid = orgUid; } - const validation = model.validateImport.validate(row, { - stripUnknown: true, - }); - if (!validation.error) { await Staging.upsert({ uuid: data[model.primaryKeyAttributes[0]], diff --git a/src/validations/labelUnit.validations.js b/src/validations/labelUnit.validations.js new file mode 100644 index 00000000..bdf33ea4 --- /dev/null +++ b/src/validations/labelUnit.validations.js @@ -0,0 +1,12 @@ +import Joi from 'joi'; + +export const labelUnitSchema = Joi.object({ + // orgUid - derived upon creation + // warehouseProjectId - derived upon creation + id: Joi.string().optional(), + orgUid: Joi.string(), + warehouseUnitId: Joi.string(), + labelId: Joi.string(), + updatedAt: Joi.date().optional(), + createdAt: Joi.date().optional(), +}); diff --git a/src/validations/labels.validations.js b/src/validations/labels.validations.js index f04dc23e..3cf56b65 100644 --- a/src/validations/labels.validations.js +++ b/src/validations/labels.validations.js @@ -1,4 +1,5 @@ import Joi from 'joi'; +import { labelUnitSchema } from './labelUnit.validations'; export const labelSchema = Joi.object({ // orgUid - derived upon creation @@ -19,4 +20,5 @@ export const labelSchema = Joi.object({ labelLink: Joi.string().required(), updatedAt: Joi.date().optional(), createdAt: Joi.date().optional(), + label_unit: labelUnitSchema.optional(), });