diff --git a/migrations/20220122164836-create-organizatin-tablec.js b/migrations/20220122164836-create-organization-table.cjs similarity index 100% rename from migrations/20220122164836-create-organizatin-tablec.js rename to migrations/20220122164836-create-organization-table.cjs diff --git a/package-lock.json b/package-lock.json index 091b254b..f9e1b5a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,7 +5,6 @@ "requires": true, "packages": { "": { - "name": "climate-warehouse", "version": "0.0.5", "hasInstallScript": true, "dependencies": { @@ -20,9 +19,9 @@ "joi": "^17.5.0", "lodash": "^4.17.21", "mysql2": "^2.3.3", + "node-xlsx": "^0.21.0", "random-hash": "^4.0.1", "request-promise": "^4.2.6", - "node-xlsx": "^0.21.0", "rxjs": "^7.5.1", "sequelize": "^6.12.0-alpha.1", "sequelize-mock": "^0.10.2", diff --git a/package.json b/package.json index fb924abc..22fbe851 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,9 @@ "joi": "^17.5.0", "lodash": "^4.17.21", "mysql2": "^2.3.3", + "node-xlsx": "^0.21.0", "random-hash": "^4.0.1", "request-promise": "^4.2.6", - "node-xlsx": "^0.21.0", "rxjs": "^7.5.1", "sequelize": "^6.12.0-alpha.1", "sequelize-mock": "^0.10.2", diff --git a/src/fullnode/simulatorV2.js b/src/fullnode/simulatorV2.js index 0c5642ea..af2e199c 100644 --- a/src/fullnode/simulatorV2.js +++ b/src/fullnode/simulatorV2.js @@ -22,12 +22,12 @@ export const pushChangeListToDataLayer = async (storeId, changeList) => { changeList.map(async (change) => { if (change.action === 'insert') { await Simulator.upsert({ - key: `simulator_${storeId}_${change.key}`, + key: `${storeId}_${change.key}`, value: change.value, }); } else if (change.action === 'delete') { await Simulator.destroy({ - where: { key: `simulator_${storeId}_${change.key}` }, + where: { key: `${storeId}_${change.key}` }, }); } }), diff --git a/src/models/projects/projects.model.js b/src/models/projects/projects.model.js index f8fcbfcb..28130d56 100644 --- a/src/models/projects/projects.model.js +++ b/src/models/projects/projects.model.js @@ -17,10 +17,13 @@ import { Qualification, ProjectLocation, CoBenefit, - Rating, + Staging, } from '../'; -import { changeListFactory } from '../../fullnode/data-layer-utils'; +import { + createXlsFromSequelizeResults, + transformFullXslsToChangeList, +} from '../../utils/xls'; import ModelTypes from './projects.modeltypes.cjs'; import { ProjectMirror } from './projects.model.mirror'; @@ -223,66 +226,72 @@ class Project extends Model { }; } - static async generateChangeListFromStagedData( - action, - warehouseProjectId, - stagedData, - ) { - const foreignKeys = [ - 'projectLocations', - 'qualifications', - 'vintages', - 'coBenefits', - 'relatedProjects', - ]; - - return Promise.resolve( - changeListFactory( - action, - warehouseProjectId, - _.omit(stagedData, foreignKeys), - ), - ); - } - - static async generateFullProjectModelChangeListFromStagedRecord(data) { - const promises = [Project.generateChangeListFromStagedData(data)]; + static generateChangeListFromStagedData(stagedData) { + const [insertRecords, updateRecords, deleteChangeList] = + Staging.seperateStagingDataIntoActionGroups(stagedData, 'Projects'); - if (data.projectLocations) { - promises.push( - ProjectLocation.generateChangeListFromStagedData(data.projectLocations), - ); - } - - if (data.qualifications) { - promises.push( - Qualification.generateChangeListFromStagedData(data.qualifications), - ); - } + const insertXslsSheets = createXlsFromSequelizeResults( + insertRecords, + Project, + false, + true, + ); - if (data.relatedProjects) { - promises.push( - Rating.generateChangeListFromStagedData(data.relatedProjects), - ); - } + const updateXslsSheets = createXlsFromSequelizeResults( + updateRecords, + Project, + false, + true, + ); - if (data.coBenefits) { - promises.push( - CoBenefit.generateChangeListFromStagedData(data.coBenefits), - ); - } + const primaryKeyMap = { + projects: 'warehouseProjectId', + projectLocations: 'id', + qualifications: 'id', + vintages: 'id', + coBenifets: 'id', + relatedProjects: 'id', + }; - if (data.vintages) { - promises.push(Vintage.generateChangeListFromStagedData(data.vintages)); - } + const insertChangeList = transformFullXslsToChangeList( + insertXslsSheets, + 'insert', + primaryKeyMap, + ); - if (data.relatedProjects) { - promises.push( - RelatedProject.generateChangeListFromStagedData(data.relatedProjects), - ); - } + const updateChangeList = transformFullXslsToChangeList( + updateXslsSheets, + 'update', + primaryKeyMap, + ); - return Promise.all(promises); + return { + projects: [ + ..._.get(insertChangeList, 'project', []), + ..._.get(updateChangeList, 'project', []), + ...deleteChangeList, + ], + qualifications: [ + ..._.get(insertChangeList, 'qualifications', []), + ..._.get(updateChangeList, 'qualifications', []), + ], + projectLocations: [ + ..._.get(insertChangeList, 'projectLocations', []), + ..._.get(updateChangeList, 'projectLocations', []), + ], + vintages: [ + ..._.get(insertChangeList, 'vintages', []), + ..._.get(updateChangeList, 'vintages', []), + ], + coBenifets: [ + ..._.get(insertChangeList, 'coBenifets', []), + ..._.get(updateChangeList, 'coBenifets', []), + ], + relatedProjects: [ + ..._.get(insertChangeList, 'relatedProjects', []), + ..._.get(updateChangeList, 'relatedProjects', []), + ], + }; } } diff --git a/src/models/qualifications/qualifications.model.js b/src/models/qualifications/qualifications.model.js index a8a70c09..648ac8b2 100644 --- a/src/models/qualifications/qualifications.model.js +++ b/src/models/qualifications/qualifications.model.js @@ -1,4 +1,5 @@ 'use strict'; + import Sequelize from 'sequelize'; const { Model } = Sequelize; import { sequelize, safeMirrorDbHandler } from '../database'; @@ -47,17 +48,6 @@ class Qualification extends Model { safeMirrorDbHandler(() => QualificationMirror.destroy(values)); return super.destroy(values); } - - static async generateChangeListFromStagedData( - // eslint-disable-next-line - action, - // eslint-disable-next-line - id, - // eslint-disable-next-line - stagedData, - ) { - return {}; - } } Qualification.init(ModelTypes, { diff --git a/src/models/staging/staging.model.js b/src/models/staging/staging.model.js index da39d540..afffb8d3 100644 --- a/src/models/staging/staging.model.js +++ b/src/models/staging/staging.model.js @@ -2,8 +2,7 @@ import Sequelize from 'sequelize'; const { Model } = Sequelize; -import { pushDataLayerChangeList } from '../../fullnode'; -import { Project, Unit, Organization } from '../../models'; +import { Project, Unit } from '../../models'; import rxjs from 'rxjs'; import { sequelize } from '../database'; @@ -23,145 +22,48 @@ class Staging extends Model { return super.destroy(values); } - static async pushToDataLayer() { - const coBenefitsChangeList = []; - const projectLocationChangeList = []; - const projectsChangeList = []; - const projectRatingChangeList = []; - const relatedProjectsChangeList = []; - const qualificationsChangeList = []; - const unitsChangeList = []; - const vintagesChangeList = []; - - const stagedChangeList = await Staging.findAll(); - - await Promise.all( - stagedChangeList.map(async (stagingRecord) => { - const { - // eslint-disable-next-line - id: stagingRecordId, - uuid, - table, - action, - commited, - data: rawData, - } = stagingRecord; - let dataSet = JSON.parse(rawData); - - await Promise.all( - dataSet.map(async (data) => { - if (table === 'Projects' && !commited) { - const [ - thisProjectChangeList, - thisProjectLocationChangeList, - thisQualificationsChangeList, - thisProjectRatingChangeList, - thisCoBenefitsChangeList, - thisVintagesChangeList, - thisRelatedProjectsChangeList, - ] = await Project.generateFullProjectModelChangeListFromStagedRecord( - action, - uuid, - data, - ); - - if (thisProjectChangeList) { - projectsChangeList.push(thisProjectChangeList); - } - - if (thisProjectLocationChangeList) { - projectLocationChangeList.push(thisProjectLocationChangeList); - } - - if (thisQualificationsChangeList) { - qualificationsChangeList.push(thisQualificationsChangeList); - } - - if (thisProjectRatingChangeList) { - projectRatingChangeList.push(thisProjectRatingChangeList); - } + static seperateStagingDataIntoActionGroups = (stagedData, table) => { + const insertRecords = []; + const updateRecords = []; + const deleteChangeList = []; + + stagedData + .filter((stagingRecord) => stagingRecord.table === table) + .forEach((stagingRecord) => { + if (stagingRecord.action === 'INSERT') { + insertRecords.push(...JSON.parse(stagingRecord.data)); + } else if (stagingRecord.action === 'UPDATE') { + updateRecords.push(...JSON.parse(stagingRecord.data)); + } else if (stagingRecord.action === 'DELETE') { + deleteChangeList.push({ + action: 'delete', + key: Buffer.from(stagingRecord.uuid).toString('hex'), + }); + } + }); + + return [insertRecords, updateRecords, deleteChangeList]; + }; - if (thisCoBenefitsChangeList) { - coBenefitsChangeList.push(thisCoBenefitsChangeList); - } - - if (thisVintagesChangeList) { - vintagesChangeList.push(thisVintagesChangeList); - } - - if (thisRelatedProjectsChangeList) { - relatedProjectsChangeList.push(thisRelatedProjectsChangeList); - } - } else if (table === 'Units' && !commited) { - const [ - thisUnitsChangeList, - thisVintagesChangeList, - thisQualificationsChangeList, - ] = await Unit.generateUnitModelChangeListFromStagedRecord( - action, - uuid, - data, - ); - - if (thisUnitsChangeList) { - unitsChangeList.push(thisUnitsChangeList); - } - - if (thisVintagesChangeList) { - vintagesChangeList.push(thisVintagesChangeList); - } - - if (thisQualificationsChangeList) { - qualificationsChangeList.push(thisQualificationsChangeList); - } - } - }), - ); - }), - ); - - const { - projectLocationStoreId, - projectRatingStoreId, - coBenefitsStoreId, - projectsStoreId, - relatedProjectsStoreId, - vintagesStoreId, - qualificationsStoreId, - // qualificationUnitJunctionStoreId, - unitsStoreId, - } = await Organization.findOne({ - where: { isHome: true }, - raw: true, - }); - - await Promise.all([ - coBenefitsChangeList.length && - pushDataLayerChangeList(coBenefitsStoreId, coBenefitsChangeList), - projectLocationChangeList.length && - pushDataLayerChangeList( - projectLocationStoreId, - projectLocationChangeList, - ), - projectsChangeList.length && - pushDataLayerChangeList(projectsStoreId, projectsChangeList), - projectRatingChangeList.length && - pushDataLayerChangeList(projectRatingStoreId, projectRatingChangeList), - relatedProjectsChangeList.length && - pushDataLayerChangeList( - relatedProjectsStoreId, - relatedProjectsChangeList, - ), - qualificationsChangeList.length && - pushDataLayerChangeList( - qualificationsStoreId, - qualificationsChangeList, - ), - unitsChangeList.length && - pushDataLayerChangeList(unitsStoreId, unitsChangeList), - vintagesChangeList.length && - pushDataLayerChangeList(vintagesStoreId, vintagesChangeList), - ]); + static async pushToDataLayer() { + const stagedRecords = await Staging.findAll({ raw: true }); + const unitsChangeList = + Unit.generateChangeListFromStagedData(stagedRecords); + + const projectsChangeList = + Project.generateChangeListFromStagedData(stagedRecords); + + const unifiedChangeList = { + ...projectsChangeList, + ...unitsChangeList, + vintages: [...unitsChangeList.vintages, ...projectsChangeList.vintages], + qualifications: [ + ...unitsChangeList.qualifications, + ...projectsChangeList.qualifications, + ], + }; + + console.log(unifiedChangeList); } } diff --git a/src/models/units/units.model.js b/src/models/units/units.model.js index f2a63bd7..22bae31d 100644 --- a/src/models/units/units.model.js +++ b/src/models/units/units.model.js @@ -8,10 +8,14 @@ import { safeMirrorDbHandler, sanitizeSqliteFtsQuery, } from '../database'; -import { Qualification, Vintage } from '../../models'; +import { Qualification, Vintage, Staging } from '../../models'; import { UnitMirror } from './units.model.mirror'; import ModelTypes from './units.modeltypes.cjs'; -import { changeListFactory } from '../../fullnode/data-layer-utils'; + +import { + createXlsFromSequelizeResults, + transformFullXslsToChangeList, +} from '../../utils/xls'; const { Model } = Sequelize; @@ -266,47 +270,61 @@ class Unit extends Model { }; } - static async generateChangeListFromStagedData( - action, - warehouseUnitId, - stagedData, - ) { - const foreignKeys = ['qualifications', 'vintage']; - if (_.get(stagedData, 'vintage.id')) { - stagedData.vintageId = stagedData.vintage.id; - } + static generateChangeListFromStagedData(stagedData) { + const [insertRecords, updateRecords, deleteChangeList] = + Staging.seperateStagingDataIntoActionGroups(stagedData, 'Units'); - return Promise.resolve( - changeListFactory( - action, - warehouseUnitId, - _.omit(stagedData, foreignKeys), - ), + const insertXslsSheets = createXlsFromSequelizeResults( + insertRecords, + Unit, + false, + true, ); - } - static generateUnitModelChangeListFromStagedRecord(action, uuid, data) { - const promises = [ - Unit.generateChangeListFromStagedData(action, uuid, data), - ]; + const updateXslsSheets = createXlsFromSequelizeResults( + updateRecords, + Unit, + false, + true, + ); - if (data.vintage) { - promises.push( - Vintage.generateChangeListFromStagedData(action, uuid, [data.vintage]), - ); - } + const primaryKeyMap = { + unit: 'warehouseUnitId', + qualifications: 'id', + qualification_units: 'qualificationunitId', + }; - if (data.qualifications) { - promises.push( - Qualification.generateChangeListFromStagedData( - action, - uuid, - data.qualifications, - ), - ); - } + const insertChangeList = transformFullXslsToChangeList( + insertXslsSheets, + 'insert', + primaryKeyMap, + ); - return Promise.all(promises); + const updateChangeList = transformFullXslsToChangeList( + updateXslsSheets, + 'update', + primaryKeyMap, + ); + + return { + units: [ + ..._.get(insertChangeList, 'unit', []), + ..._.get(updateChangeList, 'unit', []), + ...deleteChangeList, + ], + qualifications: [ + ..._.get(insertChangeList, 'qualifications', []), + ..._.get(updateChangeList, 'qualifications', []), + ], + vintages: [ + ..._.get(insertChangeList, 'vintages', []), + ..._.get(updateChangeList, 'vintages', []), + ], + qualificationUnits: [ + ..._.get(insertChangeList, 'qualification_units', []), + ..._.get(updateChangeList, 'qualification_units', []), + ], + }; } } diff --git a/src/utils/xls.js b/src/utils/xls.js index 6aee1b37..c5cf953b 100644 --- a/src/utils/xls.js +++ b/src/utils/xls.js @@ -139,6 +139,69 @@ export const createXlsFromSequelizeResults = ( if (!csv) { return xlsx.build(Object.values(xlsData)); } else { - return Object.values(xlsData).map(({ data }) => data); + return xlsData; } }; + +const checkArrayOfArrays = (a) => { + return a.every(function (x) { + return Array.isArray(x); + }); +}; + +export const transformFullXslsToChangeList = ( + xsls, + action, + primaryKeyNames, +) => { + const models = Object.keys(primaryKeyNames); + const changeList = {}; + + models.forEach((key) => { + let sheet = xsls[key]; + if (sheet) { + const headerRow = sheet.data[0]; + const primaryKeyIndex = headerRow.findIndex((item) => { + return item === primaryKeyNames[key]; + }); + + changeList[key] = []; + + sheet.data.forEach((row) => { + const rows = checkArrayOfArrays(row) ? row : [row]; + return ( + rows + // filter out the header row + .filter((r) => r[primaryKeyIndex] !== headerRow[primaryKeyIndex]) + .forEach((r) => { + if (action === 'update') { + changeList[key].push( + { + action: 'delete', + id: Buffer.from(r[primaryKeyIndex]).toString('hex'), + }, + { + action: 'insert', + id: Buffer.from(r[primaryKeyIndex]).toString('hex'), + data: Buffer.from( + r.filter((x) => typeof x === 'string').join(','), + ).toString('hex'), + }, + ); + } else { + changeList[key].push({ + action: action, + id: Buffer.from(r[primaryKeyIndex]).toString('hex'), + data: Buffer.from( + r.filter((x) => typeof x === 'string').join(','), + ).toString('hex'), + }); + } + }) + ); + }); + } + }); + + return changeList; +};