diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js index 7f8a7e07..db8b371c 100644 --- a/src/controllers/project.controller.js +++ b/src/controllers/project.controller.js @@ -112,12 +112,19 @@ export const findAll = async (req, res) => { // Remove any unsupported columns columns = columns.filter((col) => Project.defaultColumns - .concat(includes.map((model) => model.name + 's')) + .concat( + includes.map( + (include) => + `${include.model.name}${include.pluralize ? 's' : ''}`, + ), + ) .includes(col), ); } else { columns = Project.defaultColumns.concat( - includes.map((model) => model.name + 's'), + includes.map( + (include) => `${include.model.name}${include.pluralize ? 's' : ''}`, + ), ); } @@ -169,7 +176,14 @@ export const findAll = async (req, res) => { } else { return sendXls( Project.name, - createXlsFromSequelizeResults(response, Project, false, false, true), + createXlsFromSequelizeResults({ + rows: response, + model: Project, + hex: false, + toStructuredCsv: false, + excludeOrgUid: true, + isUserFriendlyFormat: true, + }), res, ); } @@ -187,7 +201,9 @@ export const findOne = async (req, res) => { const query = { where: { warehouseProjectId: req.query.warehouseProjectId }, - include: Project.getAssociatedModels(), + include: Project.getAssociatedModels().map( + (association) => association.model, + ), }; res.json(await Project.findOne(query)); diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js index 55d763e3..79244471 100644 --- a/src/controllers/units.controller.js +++ b/src/controllers/units.controller.js @@ -114,18 +114,25 @@ export const findAll = async (req, res) => { let { page, limit, columns, orgUid, search, xls } = req.query; let where = orgUid ? { orgUid } : undefined; - const includes = [Label, Issuance]; + const includes = Unit.getAssociatedModels(); if (columns) { // Remove any unsupported columns columns = columns.filter((col) => Unit.defaultColumns - .concat(includes.map((model) => model.name + 's')) + .concat( + includes.map( + (include) => + `${include.model.name}${include.pluralize ? 's' : ''}`, + ), + ) .includes(col), ); } else { columns = Unit.defaultColumns.concat( - includes.map((model) => model.name + 's'), + includes.map( + (include) => `${include.model.name}${include.pluralize ? 's' : ''}`, + ), ); } @@ -178,7 +185,14 @@ export const findAll = async (req, res) => { } else { return sendXls( Unit.name, - createXlsFromSequelizeResults(response, Unit, false, false, true), + createXlsFromSequelizeResults({ + rows: response, + model: Unit, + hex: false, + toStructuredCsv: false, + excludeOrgUid: true, + isUserFriendlyFormat: true, + }), res, ); } @@ -196,7 +210,9 @@ export const findOne = async (req, res) => { await assertDataLayerAvailable(); res.json( await Unit.findByPk(req.query.warehouseUnitId, { - include: Unit.getAssociatedModels(), + include: Unit.getAssociatedModels().map( + (association) => association.model, + ), }), ); } catch (error) { diff --git a/src/datalayer/syncService.js b/src/datalayer/syncService.js index 35e84a18..27904f4c 100644 --- a/src/datalayer/syncService.js +++ b/src/datalayer/syncService.js @@ -49,17 +49,17 @@ const syncDataLayerStoreToClimateWarehouse = async (storeId, rootHash) => { { where: { registryId: storeId } }, ); - const organizationToTrucate = await Organization.findOne({ + const organizationToTruncate = await Organization.findOne({ attributes: ['orgUid'], where: { registryId: storeId }, raw: true, }); try { - if (_.get(organizationToTrucate, 'orgUid')) { + if (_.get(organizationToTruncate, 'orgUid')) { const truncateOrganizationPromises = Object.keys(ModelKeys).map((key) => ModelKeys[key].destroy({ - where: { orgUid: organizationToTrucate.orgUid }, + where: { orgUid: organizationToTruncate.orgUid }, }), ); diff --git a/src/models/projects/projects.model.js b/src/models/projects/projects.model.js index 5a69aaec..8fc0a2a8 100644 --- a/src/models/projects/projects.model.js +++ b/src/models/projects/projects.model.js @@ -30,6 +30,7 @@ import { import ModelTypes from './projects.modeltypes.cjs'; import { ProjectMirror } from './projects.model.mirror'; import { projectsUpdateSchema } from '../../validations/index'; +import { columnsToInclude } from '../../utils/helpers.js'; class Project extends Model { static stagingTableName = 'Projects'; @@ -39,13 +40,34 @@ class Project extends Model { static virtualFieldList = {}; static getAssociatedModels = () => [ - ProjectLocation, - Label, - Issuance, - CoBenefit, - RelatedProject, - Rating, - Estimation, + { + model: ProjectLocation, + pluralize: true, + }, + { + model: Label, + pluralize: true, + }, + { + model: Issuance, + pluralize: true, + }, + { + model: CoBenefit, + pluralize: true, + }, + { + model: RelatedProject, + pluralize: true, + }, + { + model: Rating, + pluralize: true, + }, + { + model: Estimation, + pluralize: true, + }, ]; static associate() { @@ -127,7 +149,12 @@ class Project extends Model { .filter( (col) => !Project.getAssociatedModels() - .map((model) => model.name + 's') + .map( + (association) => + `${association.model.name}${ + association.pluralize ? 's' : '' + }`, + ) .includes(col), ), ); @@ -250,20 +277,6 @@ class Project extends Model { const [insertRecords, updateRecords, deleteChangeList] = Staging.seperateStagingDataIntoActionGroups(stagedData, 'Projects'); - const insertXslsSheets = createXlsFromSequelizeResults( - insertRecords, - Project, - false, - true, - ); - - const updateXslsSheets = createXlsFromSequelizeResults( - updateRecords, - Project, - false, - true, - ); - const primaryKeyMap = { project: 'warehouseProjectId', projectLocations: 'id', @@ -275,6 +288,35 @@ class Project extends Model { projectRatings: 'id', }; + const deletedRecords = await getDeletedItems(updateRecords, primaryKeyMap); + + const insertXslsSheets = createXlsFromSequelizeResults({ + rows: insertRecords, + model: Project, + hex: false, + toStructuredCsv: true, + excludeOrgUid: false, + isUserFriendlyFormat: false, + }); + + const updateXslsSheets = createXlsFromSequelizeResults({ + rows: updateRecords, + model: Project, + hex: false, + toStructuredCsv: true, + excludeOrgUid: false, + isUserFriendlyFormat: false, + }); + + const deleteXslsSheets = createXlsFromSequelizeResults({ + rows: deletedRecords, + model: Project, + hex: false, + toStructuredCsv: true, + excludeOrgUid: false, + isUserFriendlyFormat: false, + }); + const insertChangeList = await transformFullXslsToChangeList( insertXslsSheets, 'insert', @@ -287,6 +329,12 @@ class Project extends Model { primaryKeyMap, ); + const deletedAssociationsChangeList = await transformFullXslsToChangeList( + deleteXslsSheets, + 'delete', + primaryKeyMap, + ); + return { projects: [ ..._.get(insertChangeList, 'project', []), @@ -296,35 +344,115 @@ class Project extends Model { labels: [ ..._.get(insertChangeList, 'labels', []), ..._.get(updateChangeList, 'labels', []), + ..._.get(deletedAssociationsChangeList, 'labels', []), ], projectLocations: [ ..._.get(insertChangeList, 'projectLocations', []), ..._.get(updateChangeList, 'projectLocations', []), + ..._.get(deletedAssociationsChangeList, 'projectLocations', []), ], issuances: [ ..._.get(insertChangeList, 'issuances', []), ..._.get(updateChangeList, 'issuances', []), + ..._.get(deletedAssociationsChangeList, 'issuances', []), ], coBenefits: [ ..._.get(insertChangeList, 'coBenefits', []), ..._.get(updateChangeList, 'coBenefits', []), + ..._.get(deletedAssociationsChangeList, 'coBenefits', []), ], relatedProjects: [ ..._.get(insertChangeList, 'relatedProjects', []), ..._.get(updateChangeList, 'relatedProjects', []), + ..._.get(deletedAssociationsChangeList, 'relatedProjects', []), ], estimations: [ ..._.get(insertChangeList, 'estimations', []), ..._.get(updateChangeList, 'estimations', []), + ..._.get(deletedAssociationsChangeList, 'estimations', []), ], projectRatings: [ ..._.get(insertChangeList, 'projectRatings', []), ..._.get(updateChangeList, 'projectRatings', []), + ..._.get(deletedAssociationsChangeList, 'projectRatings', []), ], }; } } +/** + * Finds the deleted sub-items (e.g. labels) + * @param updatedItems {Array} - The projects updated by the user + * @param primaryKeyMap {Object} - Object map containing the primary keys for all tables + */ +async function getDeletedItems(updatedItems, primaryKeyMap) { + const updatedProductIds = updatedItems + .map((record) => record[primaryKeyMap['project']]) + .filter(Boolean); + const associations = Project.getAssociatedModels(); + + let originalProjects = []; + if (updatedProductIds.length > 0) { + const columns = [primaryKeyMap['project']].concat( + associations.map( + (association) => + `${association.model.name}${association.pluralize ? 's' : ''}`, + ), + ); + + const query = { + ...columnsToInclude(columns, associations), + }; + + const op = Sequelize.Op; + originalProjects = await Project.findAll({ + where: { + [primaryKeyMap['project']]: { + [op.in]: updatedProductIds, + }, + }, + ...query, + }); + } + + const associatedColumns = associations.map( + (association) => + `${association.model.name}${association.pluralize ? 's' : ''}`, + ); + + return originalProjects.map((originalItem) => { + const result = { ...originalItem.dataValues }; + + const updatedItem = updatedItems.find( + (item) => + item[primaryKeyMap['project']] === + originalItem[primaryKeyMap['project']], + ); + if (updatedItem == null) return; + + associatedColumns.forEach((column) => { + if (originalItem[column] == null || !Array.isArray(originalItem[column])) + return; + if (updatedItem[column] == null || !Array.isArray(updatedItem[column])) + return; + + result[column] = [...originalItem[column]]; + for (let index = originalItem[column].length - 1; index >= 0; --index) { + const item = originalItem[column][index]; + if ( + updatedItem[column].findIndex( + (searchedItem) => + searchedItem[primaryKeyMap[column]] === + item[primaryKeyMap[column]], + ) >= 0 + ) + result[column].splice(index, 1); + } + }); + return result; + }); +} + Project.init(ModelTypes, { sequelize, modelName: 'project', diff --git a/src/models/staging/staging.model.js b/src/models/staging/staging.model.js index 5a2fee9e..79356687 100644 --- a/src/models/staging/staging.model.js +++ b/src/models/staging/staging.model.js @@ -37,16 +37,14 @@ class Staging extends Model { static cleanUpCommitedAndInvalidRecords = async () => { const stagingRecords = await Staging.findAll({ raw: true }); - const stagingRecordsToDelete = await stagingRecords.filter( - async (record) => { - if (record.commited === 1) { - const { uuid, table, action, data } = record; - const diff = await Staging.getDiffObject(uuid, table, action, data); - return diff.original === null; - } - return false; - }, - ); + const stagingRecordsToDelete = stagingRecords.filter((record) => { + if (record.commited === 1) { + const { uuid, table, action, data } = record; + const diff = Staging.getDiffObject(uuid, table, action, data); + return diff.original === null; + } + return false; + }); await Staging.destroy({ where: { uuid: stagingRecordsToDelete.map((record) => record.uuid) }, @@ -62,17 +60,30 @@ class Staging extends Model { if (action === 'UPDATE') { let original; + if (table === 'Projects') { original = await Project.findOne({ where: { warehouseProjectId: uuid }, - include: Project.getAssociatedModels(), + include: Project.getAssociatedModels().map((association) => { + return { + model: association.model, + as: `${association.model.name}${ + association.pluralize ? 's' : '' + }`, + }; + }), }); - } - - if (table === 'Units') { + } else if (table === 'Units') { original = await Unit.findOne({ where: { warehouseUnitId: uuid }, - include: Unit.getAssociatedModels(), + include: Unit.getAssociatedModels().map((association) => { + return { + model: association.model, + as: `${association.model.name}${ + association.pluralize ? 's' : '' + }`, + }; + }), }); } @@ -82,17 +93,30 @@ class Staging extends Model { if (action === 'DELETE') { let original; + if (table === 'Projects') { original = await Project.findOne({ where: { warehouseProjectId: uuid }, - include: Project.getAssociatedModels(), + include: Project.getAssociatedModels().map((association) => { + return { + model: association.model, + as: `${association.model.name}${ + association.pluralize ? 's' : '' + }`, + }; + }), }); - } - - if (table === 'Units') { + } else if (table === 'Units') { original = await Unit.findOne({ where: { warehouseUnitId: uuid }, - include: Unit.getAssociatedModels(), + include: Unit.getAssociatedModels().map((association) => { + return { + model: association.model, + as: `${association.model.name}${ + association.pluralize ? 's' : '' + }`, + }; + }), }); } diff --git a/src/models/units/units.model.js b/src/models/units/units.model.js index a5a0324c..29e14f69 100644 --- a/src/models/units/units.model.js +++ b/src/models/units/units.model.js @@ -17,6 +17,7 @@ import { transformFullXslsToChangeList, } from '../../utils/xls'; import { unitsUpdateSchema } from '../../validations/index.js'; +import { columnsToInclude } from '../../utils/helpers.js'; const { Model } = Sequelize; @@ -69,9 +70,12 @@ class Unit extends Model { static getAssociatedModels = () => [ { model: Label, - as: 'labels', + pluralize: true, + }, + { + model: Issuance, + pluralize: false, }, - Issuance, ]; static associate() { @@ -291,20 +295,6 @@ class Unit extends Model { const [insertRecords, updateRecords, deleteChangeList] = Staging.seperateStagingDataIntoActionGroups(stagedData, 'Units'); - const insertXslsSheets = createXlsFromSequelizeResults( - insertRecords, - Unit, - false, - true, - ); - - const updateXslsSheets = createXlsFromSequelizeResults( - updateRecords, - Unit, - false, - true, - ); - const primaryKeyMap = { unit: 'warehouseUnitId', labels: 'id', @@ -312,6 +302,35 @@ class Unit extends Model { issuances: 'id', }; + const deletedRecords = await getDeletedItems(updateRecords, primaryKeyMap); + + const insertXslsSheets = createXlsFromSequelizeResults({ + rows: insertRecords, + model: Unit, + hex: false, + toStructuredCsv: true, + excludeOrgUid: false, + isUserFriendlyFormat: false, + }); + + const updateXslsSheets = createXlsFromSequelizeResults({ + rows: updateRecords, + model: Unit, + hex: false, + toStructuredCsv: true, + excludeOrgUid: false, + isUserFriendlyFormat: false, + }); + + const deleteXslsSheets = createXlsFromSequelizeResults({ + rows: deletedRecords, + model: Unit, + hex: false, + toStructuredCsv: true, + excludeOrgUid: false, + isUserFriendlyFormat: false, + }); + const insertChangeList = await transformFullXslsToChangeList( insertXslsSheets, 'insert', @@ -324,6 +343,12 @@ class Unit extends Model { primaryKeyMap, ); + const deletedAssociationsChangeList = await transformFullXslsToChangeList( + deleteXslsSheets, + 'delete', + primaryKeyMap, + ); + return { units: [ ..._.get(insertChangeList, 'unit', []), @@ -333,19 +358,94 @@ class Unit extends Model { labels: [ ..._.get(insertChangeList, 'labels', []), ..._.get(updateChangeList, 'labels', []), + ..._.get(deletedAssociationsChangeList, 'labels', []), ], issuances: [ ..._.get(insertChangeList, 'issuances', []), ..._.get(updateChangeList, 'issuances', []), + ..._.get(deletedAssociationsChangeList, 'issuances', []), ], labelUnits: [ ..._.get(insertChangeList, 'label_units', []), ..._.get(updateChangeList, 'label_units', []), + ..._.get(deletedAssociationsChangeList, 'label_units', []), ], }; } } +/** + * Finds the deleted sub-items (e.g. labels) + * @param updatedItems {Array} - The projects updated by the user + * @param primaryKeyMap {Object} - Object map containing the primary keys for all tables + */ +async function getDeletedItems(updatedItems, primaryKeyMap) { + const updatedUnitIds = updatedItems + .map((record) => record[primaryKeyMap['unit']]) + .filter(Boolean); + + let originalProjects = []; + if (updatedUnitIds.length > 0) { + const includes = Unit.getAssociatedModels(); + + const columns = [primaryKeyMap['unit']].concat( + includes.map( + (include) => `${include.model.name}${include.pluralize ? 's' : ''}`, + ), + ); + + const query = { + ...columnsToInclude(columns, includes), + }; + + const op = Sequelize.Op; + originalProjects = await Unit.findAll({ + where: { + [primaryKeyMap['unit']]: { + [op.in]: updatedUnitIds, + }, + }, + ...query, + }); + } + + const associatedColumns = Unit.getAssociatedModels().map( + (association) => + `${association.model.name}${association.pluralize ? 's' : ''}`, + ); + + return originalProjects.map((originalItem) => { + const result = { ...originalItem.dataValues }; + + const updatedItem = updatedItems.find( + (item) => + item[primaryKeyMap['unit']] === originalItem[primaryKeyMap['unit']], + ); + if (updatedItem == null) return; + + associatedColumns.forEach((column) => { + if (originalItem[column] == null || !Array.isArray(originalItem[column])) + return; + if (updatedItem[column] == null || !Array.isArray(updatedItem[column])) + return; + + result[column] = [...originalItem[column]]; + for (let index = originalItem[column].length - 1; index >= 0; --index) { + const item = originalItem[column][index]; + if ( + updatedItem[column].findIndex( + (searchedItem) => + searchedItem[primaryKeyMap[column]] === + item[primaryKeyMap[column]], + ) >= 0 + ) + result[column].splice(index, 1); + } + }); + return result; + }); +} + Unit.init(Object.assign({}, ModelTypes, virtualFields), { sequelize, modelName: 'unit', diff --git a/src/utils/csv-utils.js b/src/utils/csv-utils.js index 2dab0958..46d2c84d 100644 --- a/src/utils/csv-utils.js +++ b/src/utils/csv-utils.js @@ -4,12 +4,13 @@ import { uuid as uuidv4 } from 'uuidv4'; import csv from 'csvtojson'; import { Readable } from 'stream'; -import { Staging, Organization, Unit, Project } from '../models'; +import { Organization, Project, Staging, Unit } from '../models'; import { assertOrgIsHomeOrg, + assertProjectRecordExists, assertUnitRecordExists, -} from '../utils/data-assertions'; +} from './data-assertions.js'; export const createUnitRecordsFromCsv = (csvFile) => { const buffer = csvFile.data; @@ -82,23 +83,22 @@ export const createProjectRecordsFromCsv = (csvFile) => { if (newRecord.warehouseProjectId) { // Fail if they supplied their own warehouseUnitId and it doesnt exist - const possibleExistingRecord = await assertUnitRecordExists( + const possibleExistingRecord = await assertProjectRecordExists( newRecord.warehouseProjectId, ); - await assertOrgIsHomeOrg(possibleExistingRecord.dataValues.orgUid); + await assertOrgIsHomeOrg(possibleExistingRecord.orgUid); } else { - // When creating new unitd assign a uuid to is so + // When creating new project assign a uuid to is so // multiple organizations will always have unique ids - const uuid = uuidv4(); - newRecord.warehouseProjectId = uuid; - - const orgUid = _.head(Object.keys(await Organization.getHomeOrg())); - newRecord.orgUid = orgUid; + newRecord.warehouseProjectId = uuidv4(); + newRecord.orgUid = (await Organization.getHomeOrg())?.orgUid; action = 'INSERT'; } + updateProjectProperties(newRecord); + const stagedData = { uuid: newRecord.warehouseProjectId, action: action, @@ -114,7 +114,8 @@ export const createProjectRecordsFromCsv = (csvFile) => { .on('done', async () => { if (recordsToCreate.length) { await Staging.bulkCreate(recordsToCreate, { - updateOnDuplicate: ['warehouseProjectId'], + logging: console.log, + updateOnDuplicate: undefined, // TODO MariusD: find a solution for this }); resolve(); @@ -124,3 +125,31 @@ export const createProjectRecordsFromCsv = (csvFile) => { }); }); }; + +function updateProjectProperties(project) { + if (typeof project !== 'object') return; + + updateProjectArrayProp(project, 'projectLocations'); + updateProjectArrayProp(project, 'labels'); + updateProjectArrayProp(project, 'issuances'); + updateProjectArrayProp(project, 'coBenefits'); + updateProjectArrayProp(project, 'relatedProjects'); + updateProjectArrayProp(project, 'projectRatings'); + updateProjectArrayProp(project, 'estimations'); +} + +function updateProjectArrayProp(project, propName) { + if ( + project == null || + !Object.prototype.hasOwnProperty.call(project, propName) || + project[propName] == null + ) + return; + + project[propName] = JSON.parse(project[propName]); + if (Array.isArray(project[propName])) { + project[propName].forEach((item) => { + item.warehouseProjectId = project.warehouseProjectId; + }); + } +} diff --git a/src/utils/data-assertions.js b/src/utils/data-assertions.js index d294fa8c..0be57ab2 100644 --- a/src/utils/data-assertions.js +++ b/src/utils/data-assertions.js @@ -105,7 +105,12 @@ export const assertUnitRecordExists = async ( customMessage, ) => { const record = await Unit.findByPk(warehouseUnitId, { - include: Unit.getAssociatedModels(), + include: Unit.getAssociatedModels().map((association) => { + return { + model: association.model, + as: `${association.model.name}${association.pluralize ? 's' : ''}`, + }; + }), }); if (!record) { throw new Error( @@ -142,7 +147,9 @@ export const assertProjectRecordExists = async ( customMessage, ) => { const record = await Project.findByPk(warehouseProjectId, { - include: Project.getAssociatedModels(), + include: Project.getAssociatedModels().map( + (association) => association.model, + ), }); if (!record) { diff --git a/src/utils/helpers.js b/src/utils/helpers.js index c3680592..433962af 100644 --- a/src/utils/helpers.js +++ b/src/utils/helpers.js @@ -2,6 +2,8 @@ import _ from 'lodash'; +import { isPluralized } from './string-utils.js'; + export const paginationParams = (page, limit) => { if (page === undefined || limit === undefined) { return { @@ -36,15 +38,27 @@ export const optionallyPaginatedResponse = ({ count, rows }, page, limit) => { } }; +/** + * + * @param userColumns {string[]} + * @param foreignKeys {{ model: Object, pluralize: boolean }[]} + * @return {{include: unknown[], attributes: *}} + */ export const columnsToInclude = (userColumns, foreignKeys) => { + // TODO MariusD: simplify const attributeModelMap = foreignKeys.map( - (relationship) => relationship.name + 's', + (relationship) => + `${relationship.model.name}${relationship.pluralize ? 's' : ''}`, ); const filteredIncludes = _.intersection(userColumns, attributeModelMap).map( (fk) => foreignKeys.find((model) => { - return model.name === fk.substring(0, fk.length - 1); + return ( + model.model.name === fk || + (isPluralized(fk) && + model.model.name === fk.substring(0, fk.length - 1)) + ); }), ); @@ -53,13 +67,13 @@ export const columnsToInclude = (userColumns, foreignKeys) => { (column) => !attributeModelMap.includes(column), ), include: filteredIncludes.map((include) => { - if (include.name === 'label') { + if (include.pluralize) { return { - model: include, - as: include.name + 's', + model: include.model, + as: include.model.name + 's', }; } - return include; + return include.model; }), }; }; diff --git a/src/utils/string-utils.js b/src/utils/string-utils.js new file mode 100644 index 00000000..4eaa53e0 --- /dev/null +++ b/src/utils/string-utils.js @@ -0,0 +1,4 @@ +export function isPluralized(name) { + if (name == null || typeof name !== 'string') return false; + return name.endsWith('s'); +} diff --git a/src/utils/xls.js b/src/utils/xls.js index 588eb45d..b7004192 100644 --- a/src/utils/xls.js +++ b/src/utils/xls.js @@ -6,17 +6,13 @@ import stream from 'stream'; import { Staging, Organization, LabelUnit, ModelKeys } from './../models'; import { sequelize } from '../models/database'; -import { assertOrgIsHomeOrg } from '../utils/data-assertions'; -import { encodeHex } from '../utils/datalayer-utils'; +import { assertOrgIsHomeOrg } from './data-assertions'; +import { encodeHex } from './datalayer-utils'; + +import { isPluralized } from './string-utils.js'; const associations = (model) => - model.getAssociatedModels().map((model) => { - if (typeof model === 'object') { - return model.model; - } else { - return model; - } - }); + model.getAssociatedModels().map((model) => model.model); const capitalize = ([firstLetter, ...restOfWord]) => firstLetter.toUpperCase() + restOfWord.join(''); @@ -36,7 +32,7 @@ export const sendXls = (name, bytes, response) => { export const encodeValue = (value, hex = false) => { // Todo: address this elsewhere (hide these columns). This is a quick fix for complex relationships in xlsx - if (value && typeof value === 'object') { + if (value != null && typeof value === 'object') { value = value.id; } @@ -51,46 +47,308 @@ export const encodeValue = (value, hex = false) => { } }; -export const createXlsFromSequelizeResults = ( - // todo recursion +/** + * Generates either an XLS or the non-built data behind the XLS (the list of rows in plain JS object) + * @param rows {Object[]} - The items to add into the XLS + * @param model {Object} - The class model to work on (e.g. Unit or Project) + * @param toStructuredCsv {Boolean} - Whether to generate an XLS/CSV + * @param excludeOrgUid {Boolean} - Whether to exclude the organization id from the result + */ +export function createXlsFromResults({ + rows, + model, + toStructuredCsv = false, + excludeOrgUid = false, +}) { + // Unsure if this is need, therefore just left the semi-deep-clone here. The assumption is that it wants to remove all functions from the prototypes. + const rowsClone = JSON.parse(JSON.stringify(rows)); // Sadly this is the best way to simplify sequelize's return shape + + const uniqueColumns = buildColumnMap(rowsClone); + + if (excludeOrgUid) { + const columns = uniqueColumns.columns.get(uniqueColumns.topLevelKey) ?? []; + columns.splice(columns.find((column) => column === 'orgUid')); + uniqueColumns.columns.set(uniqueColumns.topLevelKey, columns); + } + + const columnTransformations = { + [model.name]: { + issuance: 'issuanceId', + }, + }; + + const primaryKey = { + [model.name]: model.primaryKeyAttributes[0], + default: 'id', + }; + + const initialValue = {}; + + const xlsData = rowsClone.reduce((aggregatedData, row) => { + return buildObjectXlsData({ + item: row, + name: model.name, + uniqueColumns: uniqueColumns, + columnsMapKey: null, + columnTransformations: columnTransformations, + primaryKeyMap: primaryKey, + primaryKeyValue: null, + parentPropName: null, + shouldPluralizeSheetName: false, + aggregatedData: aggregatedData, + }); + }, initialValue); + + return toStructuredCsv ? xlsData : xlsx.build(Object.values(xlsData)); +} + +/** + * Recursively builds the XLS data for a single item. This function could be returned from an {@ref Array.reduce} function. + * @param item {Object | Object[]} - The item to build the XLS data for + * @param name {string} - The name of the parent property for this item or a custom name. Used to generate the sheet name and to get the available transformations, primary key and other mappings + * @param uniqueColumns {{ columns: Map, topLevelKey: string }} - The list of all columns/props for each property name (retrieved using {@ref buildColumnMap} + * @param columnsMapKey {string} - The key from {@param uniqueColumns} that contains the list of columns/props for the current item + * @param columnTransformations {Object} - The mapping for column/prop name transformation for all items and sub-objects (mapped with the {@param name} prop) + * @param primaryKeyMap {Object} - The mapping for the name of the primary key for all items and sub-objects (mapped with the {@param name} prop) + * @param primaryKeyValue {unknown} - The value of the prop corresponding to the primary key for the parent item + * @param parentPropName {string} - The prop name of the parent object (the name of the prop that the parent item is bound to. The parent of the parent) + * @param shouldPluralizeSheetName {boolean} - Whether the sheet name should be expressed as plural or not + * @param aggregatedData {Object} - The object containing the resulting XLS data. Will also be returned back, to be able to use this function inside the {@ref Array.reduce} function + * @return The XLS data for a single item + */ +function buildObjectXlsData({ + item, + name, + uniqueColumns, + columnsMapKey, + columnTransformations, + primaryKeyMap, + primaryKeyValue, + parentPropName, + shouldPluralizeSheetName, + aggregatedData, +}) { + // There are too many exceptions and special rules + const columnsWithSpecialTreatment = { unit: ['issuance'] }; + + const sheetName = + !shouldPluralizeSheetName || isPluralized(name) ? name : `${name}s`; + const primaryKeyProp = primaryKeyMap[name] ?? primaryKeyMap['default']; + + const columns = + uniqueColumns.columns.get(columnsMapKey ?? uniqueColumns.topLevelKey) ?? []; + const transformations = + columnTransformations[name ?? uniqueColumns.topLevelKey] ?? {}; + + const excludedColumns = []; + + // Exclude property names that map to objects or arrays + if (item != null && typeof item === 'object' && !Array.isArray(item)) { + excludedColumns.push( + ...Object.entries(item) + .map(([column, value]) => + parentPropName == null && value != null && typeof value === 'object' + ? column + : undefined, + ) + .filter(Boolean), + ); + } + + // Special case for Unit issuance. This shouldn't exist, but it has far too many special cases. + columnsWithSpecialTreatment[name]?.forEach((specialColumn) => { + if (excludedColumns.includes(specialColumn)) { + const specialColumnIndex = excludedColumns.indexOf(specialColumn); + if (specialColumnIndex >= 0) + excludedColumns.splice(specialColumnIndex, 1); + } + }); + + // Insert a new sheet if needed + if (aggregatedData[sheetName] == null) { + aggregatedData[sheetName] = { + name: isPluralized(name) ? name : `${name}s`, + data: [ + columns + .filter((colName) => !excludedColumns.includes(colName)) + .map((colName) => transformations[colName] ?? colName), + ], + }; + + // If the primary key value of the parent item was sent, also add the name of the parent key name, suffixed by 'Id' + if (primaryKeyValue != null) { + let singularIdName = ( + isPluralized(parentPropName) + ? parentPropName.slice(0, -1) + : parentPropName + ) + .replace('_', '') + .concat('Id'); + if (aggregatedData[sheetName].data[0].includes(singularIdName)) + singularIdName = (isPluralized(name) ? name.slice(0, -1) : name) + .replace('_', '') + .concat('Id'); + + aggregatedData[sheetName].data[0].push(singularIdName); + } + } + + const xlsRowData = []; + + columns.forEach((column) => { + const itemValue = item[column]; + + // Recursively call this same function for all child items + if (itemValue != null && typeof itemValue === 'object') { + const primaryKeyName = + columnsWithSpecialTreatment[name] == null || + !columnsWithSpecialTreatment[name].includes(column) + ? primaryKeyProp + : primaryKeyMap[column] ?? primaryKeyMap['default']; + + if (!Array.isArray(itemValue)) { + const primaryKeyValue = + columnsWithSpecialTreatment[name] == null || + !columnsWithSpecialTreatment[name].includes(column) + ? item[primaryKeyName] + : itemValue[primaryKeyName]; + + buildObjectXlsData({ + item: itemValue, + name: column, + uniqueColumns: uniqueColumns, + columnsMapKey: column, + aggregatedData: aggregatedData, + primaryKeyMap: primaryKeyMap, + primaryKeyValue: primaryKeyValue, + parentPropName: name, + shouldPluralizeSheetName: true, + columnTransformations: columnTransformations, + }); + } else { + itemValue.forEach((val) => { + const primaryKeyValue = + columnsWithSpecialTreatment[name] == null || + !columnsWithSpecialTreatment[name].includes(column) + ? item[primaryKeyName] + : val[primaryKeyName]; + + if (val != null && typeof val === 'object') { + buildObjectXlsData({ + item: val, + name: column, + uniqueColumns: uniqueColumns, + columnsMapKey: column, + aggregatedData: aggregatedData, + primaryKeyMap: primaryKeyMap, + primaryKeyValue: primaryKeyValue, + parentPropName: name, + shouldPluralizeSheetName: true, + columnTransformations: columnTransformations, + }); + } + }); + } + + if ( + parentPropName == null && + (columnsWithSpecialTreatment[name] == null || + !columnsWithSpecialTreatment[name].includes(column)) + ) + return; + } + + if (itemValue != null && typeof itemValue === 'object') { + // Add the id of the child item as well if the child item is an object + const valuePrimaryKeyProp = + primaryKeyMap[column] ?? primaryKeyMap['default']; + xlsRowData.push(encodeValue(itemValue[valuePrimaryKeyProp], false)); + } else { + // Add the value of current item to the sheet if the item is not an object + xlsRowData.push(encodeValue(itemValue, false)); + } + }); + + // Also add the primary key value of the parent item + if (primaryKeyValue != null) { + xlsRowData.push(encodeValue(primaryKeyValue, false)); + } + + if (xlsRowData.length) { + aggregatedData[sheetName].data.push(xlsRowData); + } + + return aggregatedData; +} + +export const createXlsFromSequelizeResults = ({ rows, model, hex = false, toStructuredCsv = false, excludeOrgUid = false, -) => { + isUserFriendlyFormat = true, +}) => { rows = JSON.parse(JSON.stringify(rows)); // Sadly this is the best way to simplify sequelize's return shape let columnsInResults = []; + const associationColumnsMap = new Map(); - if (rows.length) { - // All rows look the same.. grab the first result to determine xls schema + if (rows.length > 0) { columnsInResults = Object.keys(rows[0]); + + rows.forEach((row, index) => { + if (index === 0) return; + + Object.keys(row).forEach((key) => { + if (!columnsInResults.includes(key)) columnsInResults.push(key); + }); + }); } - let associations = model.getAssociatedModels().map((model) => { - if (typeof model === 'object') { - return model.model; - } else { - return model; - } - }); + const associations = model.getAssociatedModels(); + const associationNames = associations.map( + (association) => `${association.model.name}s`, + ); + const columnsInMainSheet = columnsInResults.filter( - (col) => !associations.map((a) => a.name + 's').includes(col), + (column) => + !associationNames.includes(column) && + (!excludeOrgUid || column !== 'orgUid'), ); - const associatedModels = columnsInResults.filter((col) => - associations.map((a) => a.name + 's').includes(col), + + const associatedModelColumns = columnsInResults.filter((column) => + associations + .map((association) => `${association.model.name}s`) + .includes(column), ); + // Create a map with the union of all keys of each association item on any row (the columns may differ, e.g. one item added, one updated) + if (rows.length > 0) { + associatedModelColumns.forEach((column) => { + rows.forEach((row) => { + if (row[column] == null || typeof row[column] !== 'object') return; + + if (Array.isArray(row[column])) { + row[column].forEach((item) => { + if (item != null && typeof item === 'object') { + getObjectColumns(item, column, associationColumnsMap); + } + }); + } else { + getObjectColumns(row[column], column, associationColumnsMap); + } + }); + }); + } + const initialReduceValue = {}; initialReduceValue[model.name] = { name: model.name + 's', data: [ - columnsInMainSheet - .filter((colName) => { - return !(excludeOrgUid && colName === 'orgUid'); - }) - .map((colName) => (colName === 'issuance' ? 'issuanceId' : colName)), // todo make this generic + columnsInMainSheet.map((colName) => + colName === 'issuance' ? 'issuanceId' : colName, + ), // todo make this generic ], }; @@ -98,113 +356,108 @@ export const createXlsFromSequelizeResults = ( let mainXlsRow = []; // Populate main sheet values - for (const [i, mainColName] of columnsInMainSheet.entries()) { - if (row[mainColName] === null) { - row[mainColName] = 'null'; - } - - if ( - Object.keys(row).includes(mainColName) && - Object.keys(row[mainColName]).includes('id') - ) { - if (!Object.keys(sheets).includes(mainColName + 's')) { - sheets[mainColName + 's'] = { - name: mainColName + 's', + columnsInMainSheet.forEach((columnName) => { + const rowValue = + isUserFriendlyFormat && row[columnName] == null + ? 'null' + : row[columnName]; + + if (rowValue != null && Object.keys(rowValue).includes('id')) { + if (!Object.keys(sheets).includes(columnName + 's')) { + sheets[columnName + 's'] = { + name: columnName + 's', data: [ - Object.keys(row[mainColName]).concat([ + Object.keys(rowValue).concat([ model.name.split('_').join('') + 'Id', ]), ], }; } - sheets[mainColName + 's'].data.push( - Object.values(row[mainColName]) + sheets[columnName + 's'].data.push( + Object.values(rowValue) .map((val1) => encodeValue(val1, hex)) - .concat([encodeValue(row[mainColName].id, hex)]), + .concat([encodeValue(rowValue.id, hex)]), ); } - if (!associations.map((singular) => singular + 's').includes(i)) { - // Todo: change to colNames[i], but also filter column headings first (for skipping assoc cols) - if (row[mainColName] === null) { - row[mainColName] = 'null'; - } - if (!(excludeOrgUid && mainColName === 'orgUid')) { - mainXlsRow.push(encodeValue(row[mainColName], hex)); - } - } - } + + mainXlsRow.push(encodeValue(rowValue, hex)); + }); if (mainXlsRow.length) { sheets[model.name].data.push(mainXlsRow); } // Populate associated data sheets - for (const associatedModel of associatedModels) { - for (const [columnName, columnValue] of Object.entries(row)) { - if ( - !columnsInMainSheet.includes(columnName) && - columnName === associatedModel - ) { - if (Array.isArray(columnValue)) { - // one to many - // eslint-disable-next-line - for (const [_i, assocColVal] of columnValue.entries()) { - const xlsRow = []; - if (!Object.keys(sheets).includes(associatedModel)) { - sheets[associatedModel] = { - name: associatedModel, - data: [Object.keys(assocColVal).concat([model.name + 'Id'])], + associatedModelColumns.forEach((column) => { + if (!Array.isArray(row[column])) return; + + row[column].forEach((value) => { + const xlsRow = []; + + if (!Object.keys(sheets).includes(column)) { + sheets[column] = { + name: column, + data: [Object.keys(value).concat([model.name + 'Id'])], + }; + } + + (associationColumnsMap.get(column) ?? Object.keys(value)).forEach( + (column) => { + const rowValue = + isUserFriendlyFormat && value[column] == null + ? 'null' + : value[column]; + + if (rowValue != null && typeof rowValue === 'object') { + if (!Object.keys(sheets).includes(column + 's')) { + const columns = + associationColumnsMap.get(column) ?? Object.keys(rowValue); + + sheets[column + 's'] = { + name: column + 's', + data: [columns.concat([column.split('_').join('') + 'Id'])], }; } - const colNames = Object.keys(assocColVal); - for (const [i, v] of Object.values(assocColVal) - .map((col) => (col === null ? 'null' : col)) - .entries()) { - if (typeof v === 'object') { - if (!Object.keys(sheets).includes(colNames[i] + 's')) { - sheets[colNames[i] + 's'] = { - name: colNames[i] + 's', - data: [ - Object.keys(v).concat([ - colNames[i].split('_').join('') + 'Id', - ]), - ], - }; - } - sheets[colNames[i] + 's'].data.push( - Object.values(v) - .map((val1) => encodeValue(val1, hex)) - .concat([encodeValue(assocColVal.id, hex)]), - ); - } - xlsRow.push(encodeValue(v, hex)); - } - if (xlsRow.length > 0) { - xlsRow.push( - encodeValue(row[model.primaryKeyAttributes[0]], hex), + + if (rowValue != null) { + const columns = + associationColumnsMap.get(column) ?? Object.keys(rowValue); + sheets[column + 's'].data.push( + columns + .map((currentCol) => encodeValue(rowValue[currentCol], hex)) + .concat([encodeValue(value.id, hex)]), ); - sheets[associatedModel].data.push(xlsRow); } } + + xlsRow.push(encodeValue(rowValue, hex)); + }, + ); + + if (xlsRow.length > 0) { + if ((model.primaryKeyAttributes?.length ?? 0) > 0) { + xlsRow.push(encodeValue(row[model.primaryKeyAttributes[0]], hex)); } - } else { - if (columnName === 'issuanceId') { - const lastRow = sheets[model.name].data.pop(); - lastRow.pop(); - lastRow.push(columnValue); - sheets[model.name].data.push(lastRow); - } + + sheets[column].data.push(xlsRow); } - } - } + }); + }); return sheets; }, initialReduceValue); + const asdsad = createXlsFromResults({ + rows, + model, + toStructuredCsv, + excludeOrgUid, + }); + if (!toStructuredCsv) { return xlsx.build(Object.values(xlsData)); } else { - return xlsData; + return asdsad; } }; @@ -255,13 +508,9 @@ export const collapseTablesData = (tableData, model) => { // Todo recursion const collapsed = { [model.name]: tableData[model.name] }; - let associations = model.getAssociatedModels().map((model) => { - if (typeof model === 'object') { - return model.model; - } else { - return model; - } - }); + let associations = model + .getAssociatedModels() + .map((association) => association.model); for (const [i] of collapsed[model.name].data.entries()) { for (const { name: association } of associations) { @@ -464,3 +713,76 @@ export const transformFullXslsToChangeList = async ( console.log(error); } }; + +/** + * Returns a Map and the top level key name with all unique columns (props) on the items themselves and any child object they contain. + * The values are string arrays containing the column names. + * The key of the map is the name of the column (prop) as found in the parent object. The main items property are under a key named 'top level'. + * @example { id, subItem: { subId }} will return: { columns: [ 'top level': [ 'id', 'subItem' ], 'subItem': [ 'subId' ]], topLevelKey: 'top level' } + * @param items {Object[] | Object} - The items to go through and extract the columns + * @returns {{ columns: Map, topLevelKey: string }} + */ +function buildColumnMap(items) { + const result = new Map(); + const topLevelKey = 'top level'; + + if (items == null || typeof items !== 'object') + return { + columns: result, + topLevelKey: topLevelKey, + }; + + if (Array.isArray(items)) getArrayColumns(items, topLevelKey, result); + else getObjectColumns(items, topLevelKey, result); + + return { + columns: result, + topLevelKey: topLevelKey, + }; +} + +/** + * Populates the association map with the union of columns from all the objects having the same property name + * @param item {Object} - The item to look into + * @param propertyName {string} - The name of the property, as defined in the parent object (will be the key for the map) + * @param columnsMap {Map>} - The map to populate + */ +function getObjectColumns(item, propertyName, columnsMap) { + if (item == null || typeof item !== 'object') return; + + if (!Array.isArray(item)) { + const currentProperties = columnsMap.get(propertyName) ?? []; + + Object.entries(item).forEach(([column, value]) => { + if (Array.isArray(value)) getArrayColumns(value, column, columnsMap); + else if (typeof value === 'object') + getObjectColumns(value, column, columnsMap); + if (!currentProperties.includes(column)) currentProperties.push(column); + }); + + columnsMap.set(propertyName, currentProperties); + } +} + +/** + * Iterates through the array and populates the association map with the union of columns from all the objects having the same property name + * @param items {unknown[]} - The items to look into + * @param propertyName {string} - The name of the property, as defined in the parent object (will be the key for the map) + * @param columnsMap {Map>} - The map to populate + */ +function getArrayColumns(items, propertyName, columnsMap) { + if (items == null || typeof items !== 'object' || !Array.isArray(items)) + return; + + items.forEach((value) => { + if (value == null) return; + + if (Array.isArray(value)) { + getArrayColumns(value, propertyName, columnsMap); + return; + } + + if (typeof value === 'object') + getObjectColumns(value, propertyName, columnsMap); + }); +}