feat: bulk db insert with batch upload

Chia-Network · Jan 17, 2022 · 26705bb · 26705bb
1 parent f2c2616
commit 26705bb
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 10 deletions.
diff --git a/src/controllers/project.controller.js b/src/controllers/project.controller.js
@@ -199,13 +199,15 @@ export const batchUpload = async (req, res) => {
   const buffer = csvFile.data;
   const stream = Readable.from(buffer.toString('utf8'));
 
+  const recordsToCreate = [];
+
   csv()
     .fromStream(stream)
     .subscribe(async (newRecord) => {
       let action = 'UPDATE';
 
       if (newRecord.warehouseProjectId) {
-        // Fail if they supplied their own warehouseUnitId and it doesnt exist
+        // Fail if they supplied their own warehouseProjectId and it doesnt exist
         const possibleExistingRecord = await assertProjectRecordExists(
           newRecord.warehouseProjectId,
         );
@@ -234,7 +236,7 @@ export const batchUpload = async (req, res) => {
         data: JSON.stringify([newRecord]),
       };
 
-      await Staging.upsert(stagedData);
+      recordsToCreate.push(stagedData);
     })
     .on('error', (error) => {
       if (!res.headersSent) {
@@ -244,9 +246,22 @@ export const batchUpload = async (req, res) => {
         });
       }
     })
-    .on('done', () => {
-      if (!res.headersSent) {
-        res.json({ message: 'CSV processing complete' });
+    .on('done', async () => {
+      if (recordsToCreate.length) {
+        await Staging.bulkCreate(recordsToCreate);
+
+        if (!res.headersSent) {
+          res.json({
+            message:
+              'CSV processing complete, your records have been added to the staging table.',
+          });
+        }
+      } else {
+        if (!res.headersSent) {
+          res
+            .status(400)
+            .json({ message: 'There were no valid records to parse' });
+        }
       }
     });
 };
diff --git a/src/controllers/units.controller.js b/src/controllers/units.controller.js
@@ -243,6 +243,8 @@ export const batchUpload = async (req, res) => {
   const buffer = csvFile.data;
   const stream = Readable.from(buffer.toString('utf8'));
 
+  const recordsToCreate = [];
+
   csv()
     .fromStream(stream)
     .subscribe(async (newRecord) => {
@@ -251,7 +253,7 @@ export const batchUpload = async (req, res) => {
       if (newRecord.warehouseUnitId) {
         // Fail if they supplied their own warehouseUnitId and it doesnt exist
         const possibleExistingRecord = await assertUnitRecordExists(
-          req.body.warehouseUnitId,
+          newRecord.warehouseUnitId,
         );
 
         assertOrgIsHomeOrg(res, possibleExistingRecord.dataValues.orgUid);
@@ -283,7 +285,7 @@ export const batchUpload = async (req, res) => {
         data: JSON.stringify([newRecord]),
       };
 
-      await Staging.upsert(stagedData);
+      recordsToCreate.push(stagedData);
     })
     .on('error', (error) => {
       if (!res.headersSent) {
@@ -293,9 +295,22 @@ export const batchUpload = async (req, res) => {
         });
       }
     })
-    .on('done', () => {
-      if (!res.headersSent) {
-        res.json({ message: 'CSV processing complete' });
+    .on('done', async () => {
+      if (recordsToCreate.length) {
+        await Staging.bulkCreate(recordsToCreate);
+
+        if (!res.headersSent) {
+          res.json({
+            message:
+              'CSV processing complete, your records have been added to the staging table.',
+          });
+        }
+      } else {
+        if (!res.headersSent) {
+          res
+            .status(400)
+            .json({ message: 'There were no valid records to parse' });
+        }
       }
     });
 };