Merge pull request #669 from Chia-Network/feature/gisdata

Feature/gisdata

src/controllers/fileStore.controller.js

@@ -0,0 +1,78 @@
+import _ from 'lodash';
+
+import crypto from 'crypto';
+import { FileStore } from '../models';
+
+export const getFileList = async (req, res) => {
+  try {
+    const files = await FileStore.getFileStoreList();
+    res.json(files);
+  } catch (error) {
+    res.status(400).json({
+      message: 'Can not retreive file list from filestore',
+      error: error.message,
+    });
+  }
+};
+
+export const deleteFile = async (req, res) => {
+  try {
+    await FileStore.deleteFileStorItem(req.params.fileId);
+    res.status(204).end();
+  } catch (error) {
+    res.status(400).json({
+      message: 'Can not delete file from filestore',
+      error: error.message,
+    });
+  }
+};
+
+export const getFile = async (req, res) => {
+  try {
+    const { fileId } = req.body;
+    const file = await FileStore.getFileStoreItem(fileId);
+    if (file) {
+      const download = Buffer.from(file.toString('utf-8'), 'base64');
+      res.end(download);
+    } else {
+      res.status(400).json({
+        message: `FileId ${fileId} not found in the filestore.`,
+      });
+    }
+  } catch (error) {
+    res.status(400).json({
+      message: 'Can not retreive file list from filestore',
+      error: error.message,
+    });
+  }
+};
+
+export const addFileToFileStore = async (req, res) => {
+  try {
+    if (_.get(req, 'files.file.data')) {
+      const { fileName } = req.body;
+      if (!fileName) {
+        throw new Error('Missing file name, can not upload file');
+      }
+      const buffer = req.files.file.data;
+      const base64File = buffer.toString('base64');
+      const SHA256 = crypto
+        .createHash('sha256')
+        .update(base64File)
+        .digest('base64');
+      await FileStore.addFileToFileStore(SHA256, fileName, base64File);
+      return res.json({
+        message:
+          'File is being added to the file store, please wait for it to confirm.',
+      });
+    } else {
+      throw new Error('Missing file data, can not upload file.');
+    }
+  } catch (error) {
+    console.trace(error);
+    res.status(400).json({
+      message: 'Can not add file to file store',
+      error: error.message,
+    });
+  }
+};

src/controllers/index.js

@@ -6,3 +6,4 @@ export * as IssuanceController from './issuance.controller';
 export * as LabelController from './label.controller';
 export * as AuditController from './audit.controller';
 export * as GovernanceController from './governance.controller';
+export * as FileStoreController from './fileStore.controller';

src/database/migrations/20220724212553-create-file-store.js

@@ -0,0 +1,41 @@
+'use strict';
+
+import { uuid as uuidv4 } from 'uuidv4';
+
+export default {
+  async up(queryInterface, Sequelize) {
+    await Promise.all([
+      queryInterface.addColumn('organizations', 'fileStoreId', {
+        type: Sequelize.STRING,
+        allowNull: true,
+      }),
+      queryInterface.addColumn('projectLocations', 'fileId', {
+        type: Sequelize.STRING,
+        allowNull: true,
+      }),
+      queryInterface.createTable('fileStore', {
+        SHA256: {
+          type: Sequelize.STRING,
+          allowNull: false,
+          unique: true,
+          defaultValue: () => uuidv4(),
+          primaryKey: true,
+        },
+        fileName: {
+          type: Sequelize.STRING,
+          unique: true,
+        },
+        data: Sequelize.STRING,
+        orgUid: Sequelize.STRING,
+      }),
+    ]);
+  },
+
+  async down(queryInterface) {
+    await Promise.all([
+      queryInterface.removeColumn('organizations', 'fileStoreId'),
+      queryInterface.removeColumn('projectLocations', 'fileId'),
+      queryInterface.dropTable('fileStore'),
+    ]);
+  },
+};

src/database/migrations/index.js

@@ -23,6 +23,7 @@ import AddSerialNumberFields from './20220504180739-add-serial-number-fields';
 import AddDescriptionFieldToProjects from './20220509125335-add-description-field-to-projects';
 import RepopulateVirtualTables from './20220515223227-re-populate-virtual-tables';
 import AddAuthorColumnToAuditTable from './20220708210357-adding-author-column-to-audit-table';
+import CreateFileStore from './20220724212553-create-file-store';
 import AddOptionalMethodology2FieldToProject from './20220721212845-add-optional-methodology2-field-to-project';
 
 export const migrations = [
@@ -130,6 +131,10 @@ export const migrations = [
     migration: AddAuthorColumnToAuditTable,
     name: '20220708210357-adding-author-column-to-audit-table',
   },
+  {
+    migration: CreateFileStore,
+    name: '20220724212553-create-file-store',
+  },
   {
     migration: AddOptionalMethodology2FieldToProject,
     name: '20220721212845-add-optional-methodology2-field-to-project',

src/datalayer/writeService.js

@@ -3,7 +3,7 @@ import _ from 'lodash';
 import * as dataLayer from './persistance';
 import wallet from './wallet';
 import * as simulator from './simulator';
-import { encodeHex, decodeHex } from '../utils/datalayer-utils';
+import { encodeHex } from '../utils/datalayer-utils';
 import { getConfig } from '../utils/config-loader';
 import { logger } from '../config/logger.cjs';
 import { Organization } from '../models';
@@ -126,23 +126,7 @@ const pushChangesWhenStoreIsAvailable = async (
     const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId);
 
     if (!hasUnconfirmedTransactions && storeExistAndIsConfirmed) {
-      logger.info(
-        `pushing to datalayer ${storeId} ${JSON.stringify(
-          changeList.map((change) => {
-            return {
-              action: change.action,
-              key: decodeHex(change.key),
-              ...(change.value && {
-                value: /{([^*]*)}/.test(decodeHex(change.value))
-                  ? JSON.parse(decodeHex(change.value))
-                  : decodeHex(change.value),
-              }),
-            };
-          }),
-          null,
-          2,
-        )}`,
-      );
+      logger.info(`pushing to datalayer ${storeId}`);
 
       const success = await dataLayer.pushChangeListToDataLayer(
         storeId,

src/models/file-store/file-store.mock.js

@@ -0,0 +1,8 @@
+import stub from './file-store.stub.json';
+
+export const MetaMock = {
+  findAll: () => stub,
+  findOne: (id) => {
+    return stub.find((record) => record.id == id);
+  },
+};

src/models/file-store/file-store.model.js

@@ -0,0 +1,164 @@
+'use strict';
+
+/*
+  We use the SHA256 hash as the unique file ID,
+  this prevents duplicate files from being uploaded to the same store.
+*/
+
+import Sequelize from 'sequelize';
+const { Model } = Sequelize;
+import { sequelize } from '../../database';
+import { Organization } from '../organizations';
+
+import datalayer from '../../datalayer';
+import { encodeHex } from '../../utils/datalayer-utils';
+
+import ModelTypes from './file-store.modeltypes.cjs';
+
+class FileStore extends Model {
+  static async addFileToFileStore(SHA256, fileName, base64File) {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      Organization.update(
+        { fileStoreId },
+        { where: { orgUid: myOrganization.orgUid } },
+      );
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    const existingFile = await FileStore.findOne({
+      where: { SHA256 },
+      attributes: ['SHA256'],
+    });
+
+    if (existingFile) {
+      throw new Error('File Already exists in the filestore');
+    }
+
+    datalayer.syncDataLayer(fileStoreId, {
+      [SHA256]: JSON.stringify({
+        name: fileName,
+        file: base64File,
+      }),
+    });
+
+    FileStore.upsert({
+      SHA256,
+      fileName,
+      data: base64File,
+      orgUid: myOrganization.orgUid,
+    });
+  }
+
+  static async getFileStoreList() {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = await datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    new Promise((resolve, reject) => {
+      datalayer.getStoreData(
+        myOrganization.fileStoreId,
+        (data) => {
+          resolve(data);
+        },
+        reject,
+      );
+    }).then((fileStore) => {
+      // Just caching this so dont await it, we dont care when it finishes
+      return Promise.all(
+        Object.keys(fileStore).map((key) => {
+          FileStore.upsert({
+            SHA256: fileStore[key].SHA256,
+            fileName: key,
+            data: fileStore[key].data,
+            orgUid: myOrganization.orgUid,
+          });
+        }),
+      );
+    });
+
+    return FileStore.findAll({
+      attributes: ['SHA256', 'fileName'],
+      raw: true,
+    });
+  }
+
+  static async deleteFileStorItem(SHA256) {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = await datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    const changeList = {
+      action: 'delete',
+      key: encodeHex(SHA256),
+    };
+
+    datalayer.pushChangesWhenStoreIsAvailable(fileStoreId, changeList);
+
+    FileStore.destroy({ where: { SHA256, orgUid: myOrganization.org } });
+  }
+
+  static async getFileStoreItem(SHA256) {
+    const myOrganization = await Organization.getHomeOrg();
+    let fileStoreId = myOrganization.fileStoreId;
+
+    if (!fileStoreId) {
+      fileStoreId = await datalayer.createDataLayerStore();
+      datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId });
+      throw new Error('New File store being created, please try again later.');
+    }
+
+    const cachedFile = await FileStore.findOne({
+      where: { SHA256 },
+      raw: true,
+    });
+
+    if (cachedFile) {
+      return cachedFile.data;
+    }
+
+    const fileStore = await new Promise((resolve, reject) => {
+      datalayer.getStoreData(
+        myOrganization.fileStoreId,
+        (data) => {
+          resolve(data);
+        },
+        () => reject(),
+      );
+    });
+
+    // Just caching this so dont await it, we dont care when it finishes
+    FileStore.upsert({
+      SHA256,
+      fileName: fileStore[SHA256].fileName,
+      data: fileStore[SHA256].data,
+    });
+
+    return fileStore[SHA256].data;
+  }
+}
+
+FileStore.init(ModelTypes, {
+  sequelize,
+  modelName: 'fileStore',
+  freezeTableName: true,
+  timestamps: false,
+  createdAt: false,
+  updatedAt: false,
+});
+
+export { FileStore };

src/models/file-store/file-store.modeltypes.cjs

@@ -0,0 +1,17 @@
+const Sequelize = require('sequelize');
+
+module.exports = {
+  // ID is SHA256 so there are no file duplications
+  SHA256: {
+    type: Sequelize.STRING,
+    allowNull: false,
+    unique: true,
+    primaryKey: true,
+  },
+  fileName: {
+    type: Sequelize.STRING,
+    unique: true,
+  },
+  data: Sequelize.STRING,
+  orgUid: Sequelize.STRING,
+};

src/models/file-store/file-store.stub.json

@@ -0,0 +1 @@
+[]

src/models/file-store/index.js

@@ -0,0 +1,2 @@
+export * from './file-store.model.js';
+export * from './file-store.mock.js';

src/models/index.js

@@ -35,6 +35,7 @@ export * from './labelUnits';
 export * from './estimations';
 export * from './audit';
 export * from './governance';
+export * from './file-store';
 
 export const ModelKeys = {
   unit: Unit,

src/models/locations/locations.modeltypes.cjs

@@ -36,6 +36,9 @@ module.exports = {
     type: Sequelize.DATE,
     defaultValue: Sequelize.NOW,
   },
+  fileId: {
+    type: Sequelize.STRING,
+  }, 
   updatedAt: {
     type: Sequelize.DATE,
     defaultValue: Sequelize.NOW,