diff --git a/src/controllers/fileStore.controller.js b/src/controllers/fileStore.controller.js new file mode 100644 index 00000000..d4b79d36 --- /dev/null +++ b/src/controllers/fileStore.controller.js @@ -0,0 +1,78 @@ +import _ from 'lodash'; + +import crypto from 'crypto'; +import { FileStore } from '../models'; + +export const getFileList = async (req, res) => { + try { + const files = await FileStore.getFileStoreList(); + res.json(files); + } catch (error) { + res.status(400).json({ + message: 'Can not retreive file list from filestore', + error: error.message, + }); + } +}; + +export const deleteFile = async (req, res) => { + try { + await FileStore.deleteFileStorItem(req.params.fileId); + res.status(204).end(); + } catch (error) { + res.status(400).json({ + message: 'Can not delete file from filestore', + error: error.message, + }); + } +}; + +export const getFile = async (req, res) => { + try { + const { fileId } = req.body; + const file = await FileStore.getFileStoreItem(fileId); + if (file) { + const download = Buffer.from(file.toString('utf-8'), 'base64'); + res.end(download); + } else { + res.status(400).json({ + message: `FileId ${fileId} not found in the filestore.`, + }); + } + } catch (error) { + res.status(400).json({ + message: 'Can not retreive file list from filestore', + error: error.message, + }); + } +}; + +export const addFileToFileStore = async (req, res) => { + try { + if (_.get(req, 'files.file.data')) { + const { fileName } = req.body; + if (!fileName) { + throw new Error('Missing file name, can not upload file'); + } + const buffer = req.files.file.data; + const base64File = buffer.toString('base64'); + const SHA256 = crypto + .createHash('sha256') + .update(base64File) + .digest('base64'); + await FileStore.addFileToFileStore(SHA256, fileName, base64File); + return res.json({ + message: + 'File is being added to the file store, please wait for it to confirm.', + }); + } else { + throw new Error('Missing file data, can not upload file.'); + } + } catch (error) { + console.trace(error); + res.status(400).json({ + message: 'Can not add file to file store', + error: error.message, + }); + } +}; diff --git a/src/controllers/index.js b/src/controllers/index.js index e127c112..18d9b601 100644 --- a/src/controllers/index.js +++ b/src/controllers/index.js @@ -6,3 +6,4 @@ export * as IssuanceController from './issuance.controller'; export * as LabelController from './label.controller'; export * as AuditController from './audit.controller'; export * as GovernanceController from './governance.controller'; +export * as FileStoreController from './fileStore.controller'; diff --git a/src/database/migrations/20220724212553-create-file-store.js b/src/database/migrations/20220724212553-create-file-store.js new file mode 100644 index 00000000..2ec25658 --- /dev/null +++ b/src/database/migrations/20220724212553-create-file-store.js @@ -0,0 +1,41 @@ +'use strict'; + +import { uuid as uuidv4 } from 'uuidv4'; + +export default { + async up(queryInterface, Sequelize) { + await Promise.all([ + queryInterface.addColumn('organizations', 'fileStoreId', { + type: Sequelize.STRING, + allowNull: true, + }), + queryInterface.addColumn('projectLocations', 'fileId', { + type: Sequelize.STRING, + allowNull: true, + }), + queryInterface.createTable('fileStore', { + SHA256: { + type: Sequelize.STRING, + allowNull: false, + unique: true, + defaultValue: () => uuidv4(), + primaryKey: true, + }, + fileName: { + type: Sequelize.STRING, + unique: true, + }, + data: Sequelize.STRING, + orgUid: Sequelize.STRING, + }), + ]); + }, + + async down(queryInterface) { + await Promise.all([ + queryInterface.removeColumn('organizations', 'fileStoreId'), + queryInterface.removeColumn('projectLocations', 'fileId'), + queryInterface.dropTable('fileStore'), + ]); + }, +}; diff --git a/src/database/migrations/index.js b/src/database/migrations/index.js index 7a42a527..cda834c0 100644 --- a/src/database/migrations/index.js +++ b/src/database/migrations/index.js @@ -23,6 +23,7 @@ import AddSerialNumberFields from './20220504180739-add-serial-number-fields'; import AddDescriptionFieldToProjects from './20220509125335-add-description-field-to-projects'; import RepopulateVirtualTables from './20220515223227-re-populate-virtual-tables'; import AddAuthorColumnToAuditTable from './20220708210357-adding-author-column-to-audit-table'; +import CreateFileStore from './20220724212553-create-file-store'; import AddOptionalMethodology2FieldToProject from './20220721212845-add-optional-methodology2-field-to-project'; export const migrations = [ @@ -130,6 +131,10 @@ export const migrations = [ migration: AddAuthorColumnToAuditTable, name: '20220708210357-adding-author-column-to-audit-table', }, + { + migration: CreateFileStore, + name: '20220724212553-create-file-store', + }, { migration: AddOptionalMethodology2FieldToProject, name: '20220721212845-add-optional-methodology2-field-to-project', diff --git a/src/datalayer/writeService.js b/src/datalayer/writeService.js index 23480991..55d43785 100644 --- a/src/datalayer/writeService.js +++ b/src/datalayer/writeService.js @@ -3,7 +3,7 @@ import _ from 'lodash'; import * as dataLayer from './persistance'; import wallet from './wallet'; import * as simulator from './simulator'; -import { encodeHex, decodeHex } from '../utils/datalayer-utils'; +import { encodeHex } from '../utils/datalayer-utils'; import { getConfig } from '../utils/config-loader'; import { logger } from '../config/logger.cjs'; import { Organization } from '../models'; @@ -126,23 +126,7 @@ const pushChangesWhenStoreIsAvailable = async ( const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId); if (!hasUnconfirmedTransactions && storeExistAndIsConfirmed) { - logger.info( - `pushing to datalayer ${storeId} ${JSON.stringify( - changeList.map((change) => { - return { - action: change.action, - key: decodeHex(change.key), - ...(change.value && { - value: /{([^*]*)}/.test(decodeHex(change.value)) - ? JSON.parse(decodeHex(change.value)) - : decodeHex(change.value), - }), - }; - }), - null, - 2, - )}`, - ); + logger.info(`pushing to datalayer ${storeId}`); const success = await dataLayer.pushChangeListToDataLayer( storeId, diff --git a/src/models/file-store/file-store.mock.js b/src/models/file-store/file-store.mock.js new file mode 100644 index 00000000..63d449f5 --- /dev/null +++ b/src/models/file-store/file-store.mock.js @@ -0,0 +1,8 @@ +import stub from './file-store.stub.json'; + +export const MetaMock = { + findAll: () => stub, + findOne: (id) => { + return stub.find((record) => record.id == id); + }, +}; diff --git a/src/models/file-store/file-store.model.js b/src/models/file-store/file-store.model.js new file mode 100644 index 00000000..eb71d53f --- /dev/null +++ b/src/models/file-store/file-store.model.js @@ -0,0 +1,164 @@ +'use strict'; + +/* + We use the SHA256 hash as the unique file ID, + this prevents duplicate files from being uploaded to the same store. +*/ + +import Sequelize from 'sequelize'; +const { Model } = Sequelize; +import { sequelize } from '../../database'; +import { Organization } from '../organizations'; + +import datalayer from '../../datalayer'; +import { encodeHex } from '../../utils/datalayer-utils'; + +import ModelTypes from './file-store.modeltypes.cjs'; + +class FileStore extends Model { + static async addFileToFileStore(SHA256, fileName, base64File) { + const myOrganization = await Organization.getHomeOrg(); + let fileStoreId = myOrganization.fileStoreId; + + if (!fileStoreId) { + fileStoreId = datalayer.createDataLayerStore(); + datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId }); + Organization.update( + { fileStoreId }, + { where: { orgUid: myOrganization.orgUid } }, + ); + throw new Error('New File store being created, please try again later.'); + } + + const existingFile = await FileStore.findOne({ + where: { SHA256 }, + attributes: ['SHA256'], + }); + + if (existingFile) { + throw new Error('File Already exists in the filestore'); + } + + datalayer.syncDataLayer(fileStoreId, { + [SHA256]: JSON.stringify({ + name: fileName, + file: base64File, + }), + }); + + FileStore.upsert({ + SHA256, + fileName, + data: base64File, + orgUid: myOrganization.orgUid, + }); + } + + static async getFileStoreList() { + const myOrganization = await Organization.getHomeOrg(); + let fileStoreId = myOrganization.fileStoreId; + + if (!fileStoreId) { + fileStoreId = await datalayer.createDataLayerStore(); + datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId }); + throw new Error('New File store being created, please try again later.'); + } + + new Promise((resolve, reject) => { + datalayer.getStoreData( + myOrganization.fileStoreId, + (data) => { + resolve(data); + }, + reject, + ); + }).then((fileStore) => { + // Just caching this so dont await it, we dont care when it finishes + return Promise.all( + Object.keys(fileStore).map((key) => { + FileStore.upsert({ + SHA256: fileStore[key].SHA256, + fileName: key, + data: fileStore[key].data, + orgUid: myOrganization.orgUid, + }); + }), + ); + }); + + return FileStore.findAll({ + attributes: ['SHA256', 'fileName'], + raw: true, + }); + } + + static async deleteFileStorItem(SHA256) { + const myOrganization = await Organization.getHomeOrg(); + let fileStoreId = myOrganization.fileStoreId; + + if (!fileStoreId) { + fileStoreId = await datalayer.createDataLayerStore(); + datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId }); + throw new Error('New File store being created, please try again later.'); + } + + const changeList = { + action: 'delete', + key: encodeHex(SHA256), + }; + + datalayer.pushChangesWhenStoreIsAvailable(fileStoreId, changeList); + + FileStore.destroy({ where: { SHA256, orgUid: myOrganization.org } }); + } + + static async getFileStoreItem(SHA256) { + const myOrganization = await Organization.getHomeOrg(); + let fileStoreId = myOrganization.fileStoreId; + + if (!fileStoreId) { + fileStoreId = await datalayer.createDataLayerStore(); + datalayer.syncDataLayer(myOrganization.orgUid, { fileStoreId }); + throw new Error('New File store being created, please try again later.'); + } + + const cachedFile = await FileStore.findOne({ + where: { SHA256 }, + raw: true, + }); + + if (cachedFile) { + return cachedFile.data; + } + + const fileStore = await new Promise((resolve, reject) => { + datalayer.getStoreData( + myOrganization.fileStoreId, + (data) => { + resolve(data); + }, + () => reject(), + ); + }); + + // Just caching this so dont await it, we dont care when it finishes + FileStore.upsert({ + SHA256, + fileName: fileStore[SHA256].fileName, + data: fileStore[SHA256].data, + }); + + return fileStore[SHA256].data; + } +} + +FileStore.init(ModelTypes, { + sequelize, + modelName: 'fileStore', + freezeTableName: true, + timestamps: false, + createdAt: false, + updatedAt: false, +}); + +export { FileStore }; diff --git a/src/models/file-store/file-store.modeltypes.cjs b/src/models/file-store/file-store.modeltypes.cjs new file mode 100644 index 00000000..5460168b --- /dev/null +++ b/src/models/file-store/file-store.modeltypes.cjs @@ -0,0 +1,17 @@ +const Sequelize = require('sequelize'); + +module.exports = { + // ID is SHA256 so there are no file duplications + SHA256: { + type: Sequelize.STRING, + allowNull: false, + unique: true, + primaryKey: true, + }, + fileName: { + type: Sequelize.STRING, + unique: true, + }, + data: Sequelize.STRING, + orgUid: Sequelize.STRING, +}; diff --git a/src/models/file-store/file-store.stub.json b/src/models/file-store/file-store.stub.json new file mode 100644 index 00000000..60b07425 --- /dev/null +++ b/src/models/file-store/file-store.stub.json @@ -0,0 +1 @@ +[] diff --git a/src/models/file-store/index.js b/src/models/file-store/index.js new file mode 100644 index 00000000..84389e3a --- /dev/null +++ b/src/models/file-store/index.js @@ -0,0 +1,2 @@ +export * from './file-store.model.js'; +export * from './file-store.mock.js'; diff --git a/src/models/index.js b/src/models/index.js index 892f19fc..ed41212f 100644 --- a/src/models/index.js +++ b/src/models/index.js @@ -35,6 +35,7 @@ export * from './labelUnits'; export * from './estimations'; export * from './audit'; export * from './governance'; +export * from './file-store'; export const ModelKeys = { unit: Unit, diff --git a/src/models/locations/locations.modeltypes.cjs b/src/models/locations/locations.modeltypes.cjs index 3bb822e6..00deb960 100644 --- a/src/models/locations/locations.modeltypes.cjs +++ b/src/models/locations/locations.modeltypes.cjs @@ -36,6 +36,9 @@ module.exports = { type: Sequelize.DATE, defaultValue: Sequelize.NOW, }, + fileId: { + type: Sequelize.STRING, + }, updatedAt: { type: Sequelize.DATE, defaultValue: Sequelize.NOW, diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index c787fd1f..b0fbe622 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -25,7 +25,14 @@ import ModelTypes from './organizations.modeltypes.cjs'; class Organization extends Model { static async getHomeOrg(includeAddress = true) { const myOrganization = await Organization.findOne({ - attributes: ['orgUid', 'name', 'icon', 'subscribed', 'registryId'], + attributes: [ + 'orgUid', + 'name', + 'icon', + 'subscribed', + 'registryId', + 'fileStoreId', + ], where: { isHome: true }, raw: true, }); @@ -84,6 +91,7 @@ class Organization extends Model { const newRegistryId = await datalayer.createDataLayerStore(); const registryVersionId = await datalayer.createDataLayerStore(); + const fileStoreId = await datalayer.createDataLayerStore(); const revertOrganizationIfFailed = async () => { logger.info('Reverting Failed Organization'); @@ -98,6 +106,7 @@ class Organization extends Model { newOrganizationId, { registryId: newRegistryId, + fileStoreId, name, icon, }, @@ -119,6 +128,7 @@ class Organization extends Model { registryId: registryVersionId, isHome: true, subscribed: USE_SIMULATOR, + fileStoreId, name, icon, }), diff --git a/src/models/organizations/organizations.modeltypes.cjs b/src/models/organizations/organizations.modeltypes.cjs index feff47a3..bc2960d1 100644 --- a/src/models/organizations/organizations.modeltypes.cjs +++ b/src/models/organizations/organizations.modeltypes.cjs @@ -1,28 +1,29 @@ -const Sequelize = require('sequelize'); - -module.exports = { - id: { - type: Sequelize.INTEGER, - primaryKey: true, - autoIncrement: true, - }, - orgUid: { - type: Sequelize.STRING, - unique: true, - }, - orgHash: Sequelize.STRING, - name: Sequelize.STRING, - icon: Sequelize.STRING, - registryId: Sequelize.STRING, - registryHash: Sequelize.STRING, - subscribed: { - type: Sequelize.BOOLEAN, - defaultValue: false, - }, - isHome: { - type: Sequelize.BOOLEAN, - defaultValue: false, - }, - createdAt: Sequelize.DATE, - updatedAt: Sequelize.DATE, -}; +const Sequelize = require('sequelize'); + +module.exports = { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + orgUid: { + type: Sequelize.STRING, + unique: true, + }, + orgHash: Sequelize.STRING, + name: Sequelize.STRING, + icon: Sequelize.STRING, + registryId: Sequelize.STRING, + registryHash: Sequelize.STRING, + fileStoreId: Sequelize.STRING, + subscribed: { + type: Sequelize.BOOLEAN, + defaultValue: false, + }, + isHome: { + type: Sequelize.BOOLEAN, + defaultValue: false, + }, + createdAt: Sequelize.DATE, + updatedAt: Sequelize.DATE, +}; diff --git a/src/routes/v1/index.js b/src/routes/v1/index.js index ee0838c7..3f4f7024 100644 --- a/src/routes/v1/index.js +++ b/src/routes/v1/index.js @@ -12,6 +12,7 @@ import { LabelRouter, AuditRouter, GovernanceRouter, + FileStoreRouter, } from './resources'; V1Router.use('/projects', ProjectRouter); @@ -22,5 +23,6 @@ V1Router.use('/issuances', IssuanceRouter); V1Router.use('/labels', LabelRouter); V1Router.use('/audit', AuditRouter); V1Router.use('/governance', GovernanceRouter); +V1Router.use('/filestore', FileStoreRouter); export { V1Router }; diff --git a/src/routes/v1/resources/filestore.js b/src/routes/v1/resources/filestore.js new file mode 100644 index 00000000..083943fe --- /dev/null +++ b/src/routes/v1/resources/filestore.js @@ -0,0 +1,32 @@ +'use strict'; + +import express from 'express'; +import joiExpress from 'express-joi-validation'; + +const validator = joiExpress.createValidator({ passError: true }); +const FileStoreRouter = express.Router(); + +import { FileStoreController } from '../../../controllers'; +import { getFileSchema } from '../../../validations'; + +FileStoreRouter.post('/get_file', validator.body(getFileSchema), (req, res) => { + return FileStoreController.getFile(req, res); +}); + +FileStoreRouter.delete( + '/delete_file', + validator.body(getFileSchema), + (req, res) => { + return FileStoreController.deleteFile(req, res); + }, +); + +FileStoreRouter.get('/get_file_list', (req, res) => { + return FileStoreController.getFileList(req, res); +}); + +FileStoreRouter.post('/add_file', (req, res) => { + return FileStoreController.addFileToFileStore(req, res); +}); + +export { FileStoreRouter }; diff --git a/src/routes/v1/resources/index.js b/src/routes/v1/resources/index.js index 4edf37c5..8286bf94 100644 --- a/src/routes/v1/resources/index.js +++ b/src/routes/v1/resources/index.js @@ -6,3 +6,4 @@ export * from './issuances'; export * from './labels'; export * from './audit'; export * from './governance'; +export * from './filestore'; diff --git a/src/validations/filestore.validations.js b/src/validations/filestore.validations.js new file mode 100644 index 00000000..7b3a3d2b --- /dev/null +++ b/src/validations/filestore.validations.js @@ -0,0 +1,6 @@ +import Joi from 'joi'; + +export const getFileSchema = Joi.object({ + // Should be the SHA256 hash of the file you want to retreive + fileId: Joi.string().required(), +}); diff --git a/src/validations/index.js b/src/validations/index.js index d9d751de..c31aaed2 100644 --- a/src/validations/index.js +++ b/src/validations/index.js @@ -12,3 +12,4 @@ export * from './units.validations'; export * from './audit.validations'; export * from './estimations.validations'; export * from './governance.validations'; +export * from './filestore.validations';