From f5e355baf600c2c7c32725852167403631fc58e0 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Tue, 15 Feb 2022 18:12:54 -0500 Subject: [PATCH] feat: add import organization api --- src/controllers/organization.controller.js | 18 ++++- src/datalayer/persistance.js | 42 ++++++++++-- src/datalayer/simulator.js | 2 +- src/datalayer/syncService.js | 66 +++++++++++++++++-- src/datalayer/writeService.js | 8 --- .../organizations/organizations.model.js | 53 +++++++++++++-- .../organizations.modeltypes.cjs | 1 + .../organizations/organizations.stub.json | 14 ++-- src/utils/datalayer-utils.js | 7 ++ tests/resources/projects.spec.js | 5 +- tests/test-fixtures/project-fixtures.js | 2 + 11 files changed, 188 insertions(+), 30 deletions(-) diff --git a/src/controllers/organization.controller.js b/src/controllers/organization.controller.js index a9ac41b8..d8f0d9aa 100644 --- a/src/controllers/organization.controller.js +++ b/src/controllers/organization.controller.js @@ -31,7 +31,23 @@ export const create = async (req, res) => { }; // eslint-disable-next-line -export const importOrg = async (req, res) => {}; +export const importOrg = async (req, res) => { + const { orgUid, ip, port } = req.body; + try { + res.json({ + message: + 'Importing and subscribing organization this can take a few mins.', + }); + + return Organization.importOrganization(orgUid, ip, port); + } catch (error) { + console.trace(error); + res.status(400).json({ + message: 'Error importing organization', + error: error.message, + }); + } +}; export const subscribeToOrganization = async (req, res) => { try { diff --git a/src/datalayer/persistance.js b/src/datalayer/persistance.js index debb6a4f..ed39b5d0 100644 --- a/src/datalayer/persistance.js +++ b/src/datalayer/persistance.js @@ -111,8 +111,8 @@ export const getRoot = async (storeId) => { try { const data = JSON.parse(response); - console.log(data); - if (data.status === 2) { + console.log(`Root for ${storeId}`, data); + if (data.status === 2 && !data.hash.includes('0x00000000000')) { return data; } @@ -138,11 +138,14 @@ export const getStoreData = async (storeId) => { const data = JSON.parse(response); if (data.success) { + console.log('Downloaded Data', data); return data; } + + console.log('&&&&', data); } - return new Error('Error getting datalayer store data'); + return false; }; export const dataLayerAvailable = async () => { @@ -158,7 +161,7 @@ export const dataLayerAvailable = async () => { const data = JSON.parse(response); - if (Object.keys(data).includes('success')) { + if (Object.keys(data).includes('success') && data.success) { return true; } @@ -167,3 +170,34 @@ export const dataLayerAvailable = async () => { return false; } }; + +export const subscribeToStoreOnDataLayer = async (storeId, ip, port) => { + const options = { + url: `${rpcUrl}/subscribe`, + body: JSON.stringify({ + id: storeId, + ip, + port, + }), + }; + + console.log('Subscribing to: ', storeId, ip, port); + + try { + const response = await request( + Object.assign({}, getBaseOptions(), options), + ); + + const data = JSON.parse(response); + + if (Object.keys(data).includes('success') && data.success) { + console.log('Successfully Subscribed: ', storeId, ip, port); + return data; + } + + return false; + } catch (error) { + console.log('Error Subscribing: ', error); + return false; + } +}; diff --git a/src/datalayer/simulator.js b/src/datalayer/simulator.js index e34a0f1f..3dcf509e 100644 --- a/src/datalayer/simulator.js +++ b/src/datalayer/simulator.js @@ -144,4 +144,4 @@ export const dataLayerAvailable = async () => { }; // eslint-disable-next-line -export const subscribeToStore = async (storeId) => {}; +export const subscribeToStoreOnDataLayer = async (storeId) => {}; diff --git a/src/datalayer/syncService.js b/src/datalayer/syncService.js index eb75eaed..0a44118b 100644 --- a/src/datalayer/syncService.js +++ b/src/datalayer/syncService.js @@ -2,7 +2,7 @@ import _ from 'lodash'; import logUpdate from 'log-update'; -import { decodeHex } from '../utils/datalayer-utils'; +import { decodeHex, decodeDataLayerResponse } from '../utils/datalayer-utils'; import { Organization, Staging, ModelKeys } from '../models'; import * as dataLayer from './persistance'; @@ -115,8 +115,6 @@ export const dataLayerWasUpdated = async () => { rootResponse = await dataLayer.getRoots(subscribedOrgIds); } - console.log(rootResponse); - if (!rootResponse.success) { return []; } @@ -127,7 +125,6 @@ export const dataLayerWasUpdated = async () => { ); if (org) { - console.log(rootHash); // store has been updated if its confirmed and the hash has changed return rootHash.status === 2 && org.registryHash != rootHash.hash; } @@ -155,3 +152,64 @@ export const dataLayerWasUpdated = async () => { return updatedStoreIds; }; + +export const subscribeToStoreOnDataLayer = async (storeId, ip, port) => { + if (process.env.USE_SIMULATOR === 'true') { + return simulator.subscribeToStoreOnDataLayer(storeId, ip, port); + } else { + return dataLayer.subscribeToStoreOnDataLayer(storeId, ip, port); + } +}; + +export const getSubscribedStoreData = async ( + storeId, + ip, + port, + alreadySubscribed = false, + retry = 0, +) => { + if (retry > 30) { + throw new Error('Max retrys exceeded, Can not subscribe to organization'); + } + + if (!alreadySubscribed) { + const response = await subscribeToStoreOnDataLayer(storeId, ip, port); + if (!response.success) { + console.log(`Retrying...`, retry + 1); + console.log('...'); + await new Promise((resolve) => setTimeout(() => resolve(), 30000)); + return getSubscribedStoreData(storeId, ip, port, false, retry + 1); + } + } + + if (process.env.USE_SIMULATOR !== 'true') { + const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId); + if (!storeExistAndIsConfirmed) { + console.log(`Retrying...`, retry + 1); + console.log('...'); + await new Promise((resolve) => setTimeout(() => resolve(), 30000)); + return getSubscribedStoreData(storeId, ip, port, true, retry + 1); + } + } + + let encodedData; + if (process.env.USE_SIMULATOR === 'true') { + encodedData = await simulator.getStoreData(storeId); + } else { + encodedData = await dataLayer.getStoreData(storeId); + } + + if (!encodedData) { + console.log(`Retrying...`, retry + 1); + console.log('...'); + await new Promise((resolve) => setTimeout(() => resolve(), 30000)); + return getSubscribedStoreData(storeId, ip, port, true, retry + 1); + } + + const decodedData = decodeDataLayerResponse(encodedData); + + return decodedData.reduce((obj, current) => { + obj[current.key] = current.value; + return obj; + }, {}); +}; diff --git a/src/datalayer/writeService.js b/src/datalayer/writeService.js index 5e288b37..0fc73133 100644 --- a/src/datalayer/writeService.js +++ b/src/datalayer/writeService.js @@ -62,14 +62,6 @@ export const pushDataLayerChangeList = (storeId, changeList) => { pushChangesWhenStoreIsAvailable(storeId, changeList); }; -export const subscribeToStore = async (storeId) => { - if (process.env.USE_SIMULATOR === 'true') { - return simulator.subscribeToStore(storeId); - } else { - return dataLayer.subscribeToStore(storeId); - } -}; - export const dataLayerAvailable = async () => { if (process.env.USE_SIMULATOR === 'true') { return simulator.dataLayerAvailable(); diff --git a/src/models/organizations/organizations.model.js b/src/models/organizations/organizations.model.js index 275f7eb1..069535ab 100644 --- a/src/models/organizations/organizations.model.js +++ b/src/models/organizations/organizations.model.js @@ -3,7 +3,12 @@ import Sequelize from 'sequelize'; const { Model } = Sequelize; import { sequelize } from '../database'; -import { createDataLayerStore, syncDataLayer } from '../../datalayer'; +import { + createDataLayerStore, + syncDataLayer, + subscribeToStoreOnDataLayer, + getSubscribedStoreData, +} from '../../datalayer'; import ModelTypes from './organizations.modeltypes.cjs'; @@ -78,8 +83,48 @@ class Organization extends Model { }; // eslint-disable-next-line - static importHomeOrganization = (orgUid) => { - throw new Error('Not implemented yet'); + static importOrganization = async (orgUid, ip, port) => { + const orgData = await getSubscribedStoreData(orgUid, ip, port); + + if (!orgData.registryId) { + throw new Error( + 'Currupted organization, no registryId on the datalayer, can not import', + ); + } + + console.log('IMPORTING REGISTRY: ', orgData.registryId); + + const registryData = await getSubscribedStoreData( + orgData.registryId, + ip, + port, + ); + + if (!registryData.v1) { + throw new Error('Organization has no registry, can not import'); + } + + console.log('IMPORTING REGISTRY V1: ', registryData.v1); + + await subscribeToStoreOnDataLayer(registryData.v1, ip, port); + + console.log({ + orgUid, + name: orgData.name, + icon: orgData.icon, + registryId: registryData.v1, + subscribed: true, + isHome: false, + }); + + await Organization.upsert({ + orgUid, + name: orgData.name, + icon: orgData.icon, + registryId: registryData.v1, + subscribed: true, + isHome: false, + }); }; // eslint-disable-next-line @@ -88,7 +133,7 @@ class Organization extends Model { if (exists) { await Organization.update({ subscribed: true }, { orgUid }); } else { - Organization.importHomeOrganization(orgUid); + Organization.importOrganization(orgUid); } }; diff --git a/src/models/organizations/organizations.modeltypes.cjs b/src/models/organizations/organizations.modeltypes.cjs index 63f821d2..feff47a3 100644 --- a/src/models/organizations/organizations.modeltypes.cjs +++ b/src/models/organizations/organizations.modeltypes.cjs @@ -10,6 +10,7 @@ module.exports = { type: Sequelize.STRING, unique: true, }, + orgHash: Sequelize.STRING, name: Sequelize.STRING, icon: Sequelize.STRING, registryId: Sequelize.STRING, diff --git a/src/models/organizations/organizations.stub.json b/src/models/organizations/organizations.stub.json index 952beb42..43ed50ae 100644 --- a/src/models/organizations/organizations.stub.json +++ b/src/models/organizations/organizations.stub.json @@ -1,18 +1,18 @@ [ { - "orgUid": "78b472ff171a20a9de44555d8df2dc1d4c44bae5f700453a1e59ea6b2609dc5c", + "orgUid": "a807e453-6524-49df-a32d-785e56cf5600", "name": "My Org", - "registryId": "c1c017b595356361c09f9d2f7720c7da637548191764c5d6df547ddd25144d58", + "registryId": "9144c974e146920088514534c89371dc269d1b894a2c7f6cedeb80b804c6cd02", "icon": "https://climate-warehouse.s3.us-west-2.amazonaws.com/public/orgs/me.svg", - "subscribed": false, - "isHome": false + "subscribed": true, + "isHome": true }, { "orgUid": "ca1e1619-d073-4283-bdbc-4ad58838f727", - "name": "Earl Registrt", + "name": "Earl Registry", "registryId": "047b74ea567f7591f463f29dcec4cab1e760ef181c006fb3865a3e20ca6144aa", "icon": "https://climate-warehouse.s3.us-west-2.amazonaws.com/public/orgs/austria.svg", - "subscribed": true, + "subscribed": false, "isHome": false }, { @@ -20,7 +20,7 @@ "name": "Kyles Org Registry", "registryId": "eb9f9440f11f2705dd97da824f6f27079cbde3ad4ff4252539f17c8f0c08a4f9", "icon": "https://climate-warehouse.s3.us-west-2.amazonaws.com/public/orgs/denmark.svg", - "subscribed": true, + "subscribed": false, "isHome": false } ] diff --git a/src/utils/datalayer-utils.js b/src/utils/datalayer-utils.js index 3e109ae5..6ea9fdb6 100644 --- a/src/utils/datalayer-utils.js +++ b/src/utils/datalayer-utils.js @@ -5,3 +5,10 @@ export const encodeHex = (str) => { export const decodeHex = (str) => { return Buffer.from(str.replace('0x', ''), 'hex').toString(); }; + +export const decodeDataLayerResponse = (data) => { + return data.keys_values.map((item) => ({ + key: decodeHex(item.key), + value: decodeHex(item.value), + })); +}; diff --git a/tests/resources/projects.spec.js b/tests/resources/projects.spec.js index f3e433e1..41ae6864 100644 --- a/tests/resources/projects.spec.js +++ b/tests/resources/projects.spec.js @@ -13,7 +13,7 @@ describe('Project Resource CRUD', function () { it('gets all the projects available', async function () { // no query params const projects = await testFixtures.getProjectByQuery(); - expect(projects.length).to.equal(8); + expect(projects.length).to.equal(11); }); it('gets all the projects filtered by orgUid', async function () { @@ -38,6 +38,7 @@ describe('Project Resource CRUD', function () { orgUid: 'a807e453-6524-49df-a32d-785e56cf560e', search: 'City of Arcata', }); + expect(projects.length).to.equal(1); }); @@ -53,6 +54,8 @@ describe('Project Resource CRUD', function () { page: 2, limit: 3, }); + + console.log(projectsPage2); expect(projectsPage2.data.length).to.equal(3); expect(projectsPage1.data).to.not.deep.equal(projectsPage2.data); diff --git a/tests/test-fixtures/project-fixtures.js b/tests/test-fixtures/project-fixtures.js index b3c9ff0e..702050aa 100644 --- a/tests/test-fixtures/project-fixtures.js +++ b/tests/test-fixtures/project-fixtures.js @@ -67,7 +67,9 @@ export const getProject = async (warehouseProjectId) => { export const getProjectByQuery = async (query = {}) => { const result = await supertest(app).get('/v1/projects').query(query); // expect(result.body).to.be.an('array'); + expect('!!!!!', result); expect(result.statusCode).to.equal(200); + return result.body; };