feat: add filestore

src/controllers/fileStore.controller.js

@@ -0,0 +1,40 @@
+import _ from 'lodash';
+
+import crypto from 'crypto';
+import { FileStore } from '../models';
+
+/*export const getFileList = async (req, res) => {
+  try {
+  } catch (error) {
+    res.status(400).json({
+      message: 'Can not retreive audit data',
+      error: error.message,
+    });
+  }
+};*/
+
+export const addFileToFileStore = async (req, res) => {
+  try {
+    if (_.get(req, 'files.file.data')) {
+      const { fileName } = req.body;
+      if (!fileName) {
+        throw new Error('Missing file name, can not upload file');
+      }
+      const buffer = req.files.file.data;
+      const base64File = buffer.toString('base64');
+      const SHA256 = crypto.createHash('sha256', base64File).digest('base64');
+      FileStore.addFileToFileStore(SHA256, fileName, base64File);
+      return res.json({
+        message:
+          'File is being added to the file store, please wait for it to confirm.',
+      });
+    } else {
+      throw new Error('Missing file data, can not upload file.');
+    }
+  } catch (error) {
+    res.status(400).json({
+      message: 'Can not add file to file store',
+      error: error.message,
+    });
+  }
+};

src/database/migrations/20220724212553-create-file-store.js

@@ -0,0 +1,41 @@
+'use strict';
+
+import { uuid as uuidv4 } from 'uuidv4';
+
+export default {
+  async up(queryInterface, Sequelize) {
+    await Promise.all([
+      queryInterface.addColumn('organization', 'fileStoreId', {
+        type: Sequelize.STRING,
+        allowNull: true,
+      }),
+      queryInterface.addColumn('projectLocations', 'fileId', {
+        type: Sequelize.STRING,
+        allowNull: true,
+      }),
+      queryInterface.createTable('fileStore', {
+        SHA256: {
+          type: Sequelize.STRING,
+          allowNull: false,
+          unique: true,
+          defaultValue: () => uuidv4(),
+          primaryKey: true,
+        },
+        fileName: {
+          type: Sequelize.STRING,
+          unique: true,
+        },
+        data: Sequelize.STRING,
+        orgUid: Sequelize.STRING,
+      }),
+    ]);
+  },
+
+  async down(queryInterface) {
+    await Promise.all([
+      queryInterface.removeColumn('organization', 'fileStoreId'),
+      queryInterface.removeColumn('projectLocations', 'fileId'),
+      queryInterface.dropTable('fileStore'),
+    ]);
+  },
+};

src/database/migrations/index.js

@@ -23,6 +23,7 @@ import AddSerialNumberFields from './20220504180739-add-serial-number-fields';
 import AddDescriptionFieldToProjects from './20220509125335-add-description-field-to-projects';
 import RepopulateVirtualTables from './20220515223227-re-populate-virtual-tables';
 import AddAuthorColumnToAuditTable from './20220708210357-adding-author-column-to-audit-table';
+import CreateFileStore from './20220724212553-create-file-store';
 
 export const migrations = [
   {
@@ -128,5 +129,9 @@ export const migrations = [
   {
     migration: AddAuthorColumnToAuditTable,
     name: '20220708210357-adding-author-column-to-audit-table',
-  }
+  },
+  {
+    migration: CreateFileStore,
+    name: '20220724212553-create-file-store',
+  },
 ];

src/models/file-store/file-store.mock.js

@@ -0,0 +1,8 @@
+import stub from './file-store.stub.json';
+
+export const MetaMock = {
+  findAll: () => stub,
+  findOne: (id) => {
+    return stub.find((record) => record.id == id);
+  },
+};

src/models/file-store/file-store.model.js

@@ -0,0 +1,128 @@
+'use strict';
+
+/*
+  We use the SHA256 hash as the unique file ID,
+  this prevents duplicate files from being uploaded to the same store.
+*/
+
+import Sequelize from 'sequelize';
+const { Model, Organization } = Sequelize;
+import { sequelize } from '../../database';
+
+import datalayer from '../../datalayer';
+
+import ModelTypes from './file-store.modeltypes.cjs';
+
+class FileStore extends Model {
+  static async addFileToFileStore(SHA256, fileName, base64File) {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = await datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    await datalayer.syncDataLayer(fileStoreId, {
+      [SHA256]: {
+        name: fileName,
+        file: base64File,
+      },
+    });
+
+    FileStore.upsert({
+      SHA256,
+      fileName,
+      data: base64File,
+    });
+  }
+
+  static async getFileStoreList() {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = await datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    new Promise((resolve, reject) => {
+      datalayer.getStoreData(
+        myOrganization.fileStoreId,
+        (data) => {
+          resolve(data);
+        },
+        reject,
+      );
+    }).then((fileStore) => {
+      // Just caching this so dont await it, we dont care when it finishes
+      return Promise.all(
+        Object.keys(fileStore).map((key) => {
+          FileStore.upsert({
+            SHA256: fileStore[key].SHA256,
+            fileName: key,
+            data: fileStore[key].data,
+            orgUid: myOrganization.orgUid,
+          });
+        }),
+      );
+    });
+
+    return FileStore.findAll({
+      attributes: ['SHA256', 'fileName'],
+      raw: true,
+    });
+  }
+
+  static async getFileStoreItem(SHA256) {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = await datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    const cachedFile = await FileStore.findOne({
+      where: { SHA256 },
+      raw: true,
+    });
+
+    if (cachedFile) {
+      return cachedFile.data;
+    }
+
+    const fileStore = await new Promise((resolve, reject) => {
+      datalayer.getStoreData(
+        myOrganization.fileStoreId,
+        (data) => {
+          resolve(data);
+        },
+        () => reject(),
+      );
+    });
+
+    // Just caching this so dont await it, we dont care when it finishes
+    FileStore.upsert({
+      SHA256,
+      fileName: fileStore[SHA256].fileName,
+      data: fileStore[SHA256].data,
+    });
+
+    return fileStore[SHA256].data;
+  }
+}
+
+FileStore.init(ModelTypes, {
+  sequelize,
+  modelName: 'fileStore',
+  freezeTableName: true,
+  timestamps: false,
+  createdAt: false,
+  updatedAt: false,
+});
+
+export { FileStore };

src/models/file-store/file-store.modeltypes.cjs

@@ -0,0 +1,17 @@
+const Sequelize = require('sequelize');
+
+module.exports = {
+  // ID is SHA256 so there are no file duplications
+  SHA256: {
+    type: Sequelize.STRING,
+    allowNull: false,
+    unique: true,
+    primaryKey: true,
+  },
+  fileName: {
+    type: Sequelize.STRING,
+    unique: true,
+  },
+  data: Sequelize.STRING,
+  orgUid: Sequelize.STRING,
+};

src/models/file-store/file-store.stub.json

@@ -0,0 +1 @@
+[]

src/models/file-store/index.js

@@ -0,0 +1,2 @@
+export * from './file-store.model.js';
+export * from './file-store.mock.js';

src/models/index.js

@@ -35,6 +35,7 @@ export * from './labelUnits';
 export * from './estimations';
 export * from './audit';
 export * from './governance';
+export * from './file-store';
 
 export const ModelKeys = {
   unit: Unit,

src/models/locations/locations.modeltypes.cjs

@@ -36,6 +36,9 @@ module.exports = {
     type: Sequelize.DATE,
     defaultValue: Sequelize.NOW,
   },
+  fileId: {
+    type: Sequelize.STRING,
+  }, 
   updatedAt: {
     type: Sequelize.DATE,
     defaultValue: Sequelize.NOW,

src/models/organizations/organizations.model.js

@@ -25,7 +25,14 @@ import ModelTypes from './organizations.modeltypes.cjs';
 class Organization extends Model {
   static async getHomeOrg(includeAddress = true) {
     const myOrganization = await Organization.findOne({
-      attributes: ['orgUid', 'name', 'icon', 'subscribed', 'registryId'],
+      attributes: [
+        'orgUid',
+        'name',
+        'icon',
+        'subscribed',
+        'registryId',
+        'fileStoreId',
+      ],
       where: { isHome: true },
       raw: true,
     });
@@ -84,6 +91,7 @@ class Organization extends Model {
 
       const newRegistryId = await datalayer.createDataLayerStore();
       const registryVersionId = await datalayer.createDataLayerStore();
+      const fileStoreId = await datalayer.createDataLayerStore();
 
       const revertOrganizationIfFailed = async () => {
         logger.info('Reverting Failed Organization');
@@ -98,6 +106,7 @@ class Organization extends Model {
         newOrganizationId,
         {
           registryId: newRegistryId,
+          fileStoreId,
           name,
           icon,
         },
@@ -119,6 +128,7 @@ class Organization extends Model {
           registryId: registryVersionId,
           isHome: true,
           subscribed: USE_SIMULATOR,
+          fileStoreId,
           name,
           icon,
         }),

src/models/organizations/organizations.modeltypes.cjs

@@ -1,28 +1,29 @@
-const Sequelize = require('sequelize');
-
-module.exports = {
-  id: {
-    type: Sequelize.INTEGER,
-    primaryKey: true,
-    autoIncrement: true,
-  },
-  orgUid: {
-    type: Sequelize.STRING,
-    unique: true,
-  },
-  orgHash: Sequelize.STRING,
-  name: Sequelize.STRING,
-  icon: Sequelize.STRING,
-  registryId: Sequelize.STRING,
-  registryHash: Sequelize.STRING,
-  subscribed: {
-    type: Sequelize.BOOLEAN,
-    defaultValue: false,
-  },
-  isHome: {
-    type: Sequelize.BOOLEAN,
-    defaultValue: false,
-  },
-  createdAt: Sequelize.DATE,
-  updatedAt: Sequelize.DATE,
-};
+const Sequelize = require('sequelize');
+
+module.exports = {
+  id: {
+    type: Sequelize.INTEGER,
+    primaryKey: true,
+    autoIncrement: true,
+  },
+  orgUid: {
+    type: Sequelize.STRING,
+    unique: true,
+  },
+  orgHash: Sequelize.STRING,
+  name: Sequelize.STRING,
+  icon: Sequelize.STRING,
+  registryId: Sequelize.STRING,
+  registryHash: Sequelize.STRING,
+  fileStoreId: Sequelize.STRING,
+  subscribed: {
+    type: Sequelize.BOOLEAN,
+    defaultValue: false,
+  },
+  isHome: {
+    type: Sequelize.BOOLEAN,
+    defaultValue: false,
+  },
+  createdAt: Sequelize.DATE,
+  updatedAt: Sequelize.DATE,
+};