refactor: rebase on develop

src/controllers/project.controller.js

@@ -112,12 +112,19 @@ export const findAll = async (req, res) => {
       // Remove any unsupported columns
       columns = columns.filter((col) =>
         Project.defaultColumns
-          .concat(includes.map((model) => model.name + 's'))
+          .concat(
+            includes.map(
+              (include) =>
+                `${include.model.name}${include.pluralize ? 's' : ''}`,
+            ),
+          )
           .includes(col),
       );
     } else {
       columns = Project.defaultColumns.concat(
-        includes.map((model) => model.name + 's'),
+        includes.map(
+          (include) => `${include.model.name}${include.pluralize ? 's' : ''}`,
+        ),
       );
     }
 
@@ -169,7 +176,14 @@ export const findAll = async (req, res) => {
     } else {
       return sendXls(
         Project.name,
-        createXlsFromSequelizeResults(response, Project, false, false, true),
+        createXlsFromSequelizeResults({
+          rows: response,
+          model: Project,
+          hex: false,
+          toStructuredCsv: false,
+          excludeOrgUid: true,
+          isUserFriendlyFormat: true,
+        }),
         res,
       );
     }
@@ -187,7 +201,9 @@ export const findOne = async (req, res) => {
 
     const query = {
       where: { warehouseProjectId: req.query.warehouseProjectId },
-      include: Project.getAssociatedModels(),
+      include: Project.getAssociatedModels().map(
+        (association) => association.model,
+      ),
     };
 
     res.json(await Project.findOne(query));

src/controllers/units.controller.js

@@ -114,18 +114,25 @@ export const findAll = async (req, res) => {
     let { page, limit, columns, orgUid, search, xls } = req.query;
     let where = orgUid ? { orgUid } : undefined;
 
-    const includes = [Label, Issuance];
+    const includes = Unit.getAssociatedModels();
 
     if (columns) {
       // Remove any unsupported columns
       columns = columns.filter((col) =>
         Unit.defaultColumns
-          .concat(includes.map((model) => model.name + 's'))
+          .concat(
+            includes.map(
+              (include) =>
+                `${include.model.name}${include.pluralize ? 's' : ''}`,
+            ),
+          )
           .includes(col),
       );
     } else {
       columns = Unit.defaultColumns.concat(
-        includes.map((model) => model.name + 's'),
+        includes.map(
+          (include) => `${include.model.name}${include.pluralize ? 's' : ''}`,
+        ),
       );
     }
 
@@ -178,7 +185,14 @@ export const findAll = async (req, res) => {
     } else {
       return sendXls(
         Unit.name,
-        createXlsFromSequelizeResults(response, Unit, false, false, true),
+        createXlsFromSequelizeResults({
+          rows: response,
+          model: Unit,
+          hex: false,
+          toStructuredCsv: false,
+          excludeOrgUid: true,
+          isUserFriendlyFormat: true,
+        }),
         res,
       );
     }
@@ -196,7 +210,9 @@ export const findOne = async (req, res) => {
     await assertDataLayerAvailable();
     res.json(
       await Unit.findByPk(req.query.warehouseUnitId, {
-        include: Unit.getAssociatedModels(),
+        include: Unit.getAssociatedModels().map(
+          (association) => association.model,
+        ),
       }),
     );
   } catch (error) {

src/datalayer/syncService.js

@@ -49,17 +49,17 @@ const syncDataLayerStoreToClimateWarehouse = async (storeId, rootHash) => {
     { where: { registryId: storeId } },
   );
 
-  const organizationToTrucate = await Organization.findOne({
+  const organizationToTruncate = await Organization.findOne({
     attributes: ['orgUid'],
     where: { registryId: storeId },
     raw: true,
   });
 
   try {
-    if (_.get(organizationToTrucate, 'orgUid')) {
+    if (_.get(organizationToTruncate, 'orgUid')) {
       const truncateOrganizationPromises = Object.keys(ModelKeys).map((key) =>
         ModelKeys[key].destroy({
-          where: { orgUid: organizationToTrucate.orgUid },
+          where: { orgUid: organizationToTruncate.orgUid },
         }),
       );
 

src/models/projects/projects.model.js

@@ -30,6 +30,7 @@ import {
 import ModelTypes from './projects.modeltypes.cjs';
 import { ProjectMirror } from './projects.model.mirror';
 import { projectsUpdateSchema } from '../../validations/index';
+import { columnsToInclude } from '../../utils/helpers.js';
 
 class Project extends Model {
   static stagingTableName = 'Projects';
@@ -39,13 +40,34 @@ class Project extends Model {
   static virtualFieldList = {};
 
   static getAssociatedModels = () => [
-    ProjectLocation,
-    Label,
-    Issuance,
-    CoBenefit,
-    RelatedProject,
-    Rating,
-    Estimation,
+    {
+      model: ProjectLocation,
+      pluralize: true,
+    },
+    {
+      model: Label,
+      pluralize: true,
+    },
+    {
+      model: Issuance,
+      pluralize: true,
+    },
+    {
+      model: CoBenefit,
+      pluralize: true,
+    },
+    {
+      model: RelatedProject,
+      pluralize: true,
+    },
+    {
+      model: Rating,
+      pluralize: true,
+    },
+    {
+      model: Estimation,
+      pluralize: true,
+    },
   ];
 
   static associate() {
@@ -127,7 +149,12 @@ class Project extends Model {
         .filter(
           (col) =>
             !Project.getAssociatedModels()
-              .map((model) => model.name + 's')
+              .map(
+                (association) =>
+                  `${association.model.name}${
+                    association.pluralize ? 's' : ''
+                  }`,
+              )
               .includes(col),
         ),
     );
@@ -250,20 +277,6 @@ class Project extends Model {
     const [insertRecords, updateRecords, deleteChangeList] =
       Staging.seperateStagingDataIntoActionGroups(stagedData, 'Projects');
 
-    const insertXslsSheets = createXlsFromSequelizeResults(
-      insertRecords,
-      Project,
-      false,
-      true,
-    );
-
-    const updateXslsSheets = createXlsFromSequelizeResults(
-      updateRecords,
-      Project,
-      false,
-      true,
-    );
-
     const primaryKeyMap = {
       project: 'warehouseProjectId',
       projectLocations: 'id',
@@ -275,6 +288,35 @@ class Project extends Model {
       projectRatings: 'id',
     };
 
+    const deletedRecords = await getDeletedItems(updateRecords, primaryKeyMap);
+
+    const insertXslsSheets = createXlsFromSequelizeResults({
+      rows: insertRecords,
+      model: Project,
+      hex: false,
+      toStructuredCsv: true,
+      excludeOrgUid: false,
+      isUserFriendlyFormat: false,
+    });
+
+    const updateXslsSheets = createXlsFromSequelizeResults({
+      rows: updateRecords,
+      model: Project,
+      hex: false,
+      toStructuredCsv: true,
+      excludeOrgUid: false,
+      isUserFriendlyFormat: false,
+    });
+
+    const deleteXslsSheets = createXlsFromSequelizeResults({
+      rows: deletedRecords,
+      model: Project,
+      hex: false,
+      toStructuredCsv: true,
+      excludeOrgUid: false,
+      isUserFriendlyFormat: false,
+    });
+
     const insertChangeList = await transformFullXslsToChangeList(
       insertXslsSheets,
       'insert',
@@ -287,6 +329,12 @@ class Project extends Model {
       primaryKeyMap,
     );
 
+    const deletedAssociationsChangeList = await transformFullXslsToChangeList(
+      deleteXslsSheets,
+      'delete',
+      primaryKeyMap,
+    );
+
     return {
       projects: [
         ..._.get(insertChangeList, 'project', []),
@@ -296,35 +344,115 @@ class Project extends Model {
       labels: [
         ..._.get(insertChangeList, 'labels', []),
         ..._.get(updateChangeList, 'labels', []),
+        ..._.get(deletedAssociationsChangeList, 'labels', []),
       ],
       projectLocations: [
         ..._.get(insertChangeList, 'projectLocations', []),
         ..._.get(updateChangeList, 'projectLocations', []),
+        ..._.get(deletedAssociationsChangeList, 'projectLocations', []),
       ],
       issuances: [
         ..._.get(insertChangeList, 'issuances', []),
         ..._.get(updateChangeList, 'issuances', []),
+        ..._.get(deletedAssociationsChangeList, 'issuances', []),
       ],
       coBenefits: [
         ..._.get(insertChangeList, 'coBenefits', []),
         ..._.get(updateChangeList, 'coBenefits', []),
+        ..._.get(deletedAssociationsChangeList, 'coBenefits', []),
       ],
       relatedProjects: [
         ..._.get(insertChangeList, 'relatedProjects', []),
         ..._.get(updateChangeList, 'relatedProjects', []),
+        ..._.get(deletedAssociationsChangeList, 'relatedProjects', []),
       ],
       estimations: [
         ..._.get(insertChangeList, 'estimations', []),
         ..._.get(updateChangeList, 'estimations', []),
+        ..._.get(deletedAssociationsChangeList, 'estimations', []),
       ],
       projectRatings: [
         ..._.get(insertChangeList, 'projectRatings', []),
         ..._.get(updateChangeList, 'projectRatings', []),
+        ..._.get(deletedAssociationsChangeList, 'projectRatings', []),
       ],
     };
   }
 }
 
+/**
+ * Finds the deleted sub-items (e.g. labels)
+ * @param updatedItems {Array} - The projects updated by the user
+ * @param primaryKeyMap {Object} - Object map containing the primary keys for all tables
+ */
+async function getDeletedItems(updatedItems, primaryKeyMap) {
+  const updatedProductIds = updatedItems
+    .map((record) => record[primaryKeyMap['project']])
+    .filter(Boolean);
+  const associations = Project.getAssociatedModels();
+
+  let originalProjects = [];
+  if (updatedProductIds.length > 0) {
+    const columns = [primaryKeyMap['project']].concat(
+      associations.map(
+        (association) =>
+          `${association.model.name}${association.pluralize ? 's' : ''}`,
+      ),
+    );
+
+    const query = {
+      ...columnsToInclude(columns, associations),
+    };
+
+    const op = Sequelize.Op;
+    originalProjects = await Project.findAll({
+      where: {
+        [primaryKeyMap['project']]: {
+          [op.in]: updatedProductIds,
+        },
+      },
+      ...query,
+    });
+  }
+
+  const associatedColumns = associations.map(
+    (association) =>
+      `${association.model.name}${association.pluralize ? 's' : ''}`,
+  );
+
+  return originalProjects.map((originalItem) => {
+    const result = { ...originalItem.dataValues };
+
+    const updatedItem = updatedItems.find(
+      (item) =>
+        item[primaryKeyMap['project']] ===
+        originalItem[primaryKeyMap['project']],
+    );
+    if (updatedItem == null) return;
+
+    associatedColumns.forEach((column) => {
+      if (originalItem[column] == null || !Array.isArray(originalItem[column]))
+        return;
+      if (updatedItem[column] == null || !Array.isArray(updatedItem[column]))
+        return;
+
+      result[column] = [...originalItem[column]];
+      for (let index = originalItem[column].length - 1; index >= 0; --index) {
+        const item = originalItem[column][index];
+        if (
+          updatedItem[column].findIndex(
+            (searchedItem) =>
+              searchedItem[primaryKeyMap[column]] ===
+              item[primaryKeyMap[column]],
+          ) >= 0
+        )
+          result[column].splice(index, 1);
+      }
+    });
+    return result;
+  });
+}
+
 Project.init(ModelTypes, {
   sequelize,
   modelName: 'project',