feat: add datamodel folders

Chia-Network · May 19, 2022 · 73bf7a3 · 73bf7a3
1 parent 5f1031a
commit 73bf7a3
Show file tree

Hide file tree

Showing 6 changed files with 186 additions and 88 deletions.
diff --git a/src/config/config.cjs b/src/config/config.cjs
@@ -10,11 +10,30 @@ const logger = require('./logger.cjs').logger;
 const homeDir = os.homedir();
 const defaultConfig = require('../utils/defaultConfig.json');
 
-const persistanceFolder = `${homeDir}/.chia/climate-warehouse`;
+const getDataModelVersion = () => {
+  const packageJson = fileLoader('package.json');
+  const version = packageJson.version;
+  const majorVersion = version.split('.')[0];
+  return `v${majorVersion}`;
+};
+
+const fileLoader = _.memoize((filepath) => {
+  logger.debug(`Reading file at ${filepath}`);
+
+  const file = path.resolve(filepath);
+
+  try {
+    return yaml.load(fs.readFileSync(file, 'utf8'));
+  } catch (e) {
+    logger.error(`File not found at ${file}`, e);
+  }
+});
+
+const persistanceFolder = `${homeDir}/.chia/climate-warehouse/${getDataModelVersion()}`;
 
 // Adding this duplicate function here because im having trouble
 // importing it in from utils folder
-const configPath = `${homeDir}/.chia/climate-warehouse/config.yaml`;
+const configPath = `${persistanceFolder}/config.yaml`;
 const getConfig = _.memoize(() => {
   logger.info(`Reading config file at ${configPath}`);
 
@@ -25,6 +44,10 @@ const getConfig = _.memoize(() => {
   // First write it to chia home
   if (!fs.existsSync(configFile)) {
     try {
+      if (!fs.existsSync(persistanceFolder)) {
+        fs.mkdirSync(persistanceFolder, { recursive: true });
+      }
+
       fs.writeFileSync(configFile, yaml.dump(defaultConfig), 'utf8');
     } catch (err) {
       // if it still doesnt exist that means we are in an env without write permissions

diff --git a/src/config/logger.cjs b/src/config/logger.cjs
@@ -1,17 +1,48 @@
+const _ = require('lodash');
 const winston = require('winston');
-const { format, transports, createLogger } = winston
-
+const { format, transports, createLogger } = winston;
+const yaml = require('js-yaml');
 const DailyRotateFile = require('winston-daily-rotate-file');
+const path = require('path');
 
 const fs = require('fs');
 const os = require('os');
 const homeDir = os.homedir();
-const logDir = `${homeDir}/.chia/climate-warehouse/logs`;
+
+const fileLoader = _.memoize((filepath) => {
+  console.log(`Reading file at ${filepath}`);
+
+  const file = path.resolve(filepath);
+
+  try {
+    return yaml.load(fs.readFileSync(file, 'utf8'));
+  } catch (e) {
+    console.log(`File not found at ${file}`, e);
+  }
+});
+
+const getDataModelVersion = () => {
+  const packageJson = fileLoader('package.json');
+  const version = packageJson.version;
+  const majorVersion = version.split('.')[0];
+  return `v${majorVersion}`;
+};
+
+
+
+const logDir = `${homeDir}/.chia/climate-warehouse/${getDataModelVersion()}/logs`;
 
 if (!fs.existsSync(logDir)) {
   fs.mkdirSync(logDir, { recursive: true });
 }
-const logFormat = format.printf(info => `${info.timestamp} [${info.level}]: ${info.message} ${Object.keys(info.metadata || {}).length > 0 ? JSON.stringify(info.metadata) : ''}`)
+const logFormat = format.printf(
+  (info) =>
+    `${info.timestamp} [${info.level}]: ${info.message} ${
+      Object.keys(info.metadata || {}).length > 0
+        ? JSON.stringify(info.metadata)
+        : ''
+    }`,
+);
 
 const logger = createLogger({
   level: 'info',
@@ -26,16 +57,12 @@ const logger = createLogger({
     new transports.File({
       filename: `${logDir}/error.log`,
       level: 'error',
-      format: format.combine(
-        format.json(),
-      ),
+      format: format.combine(format.json()),
     }),
     // Write all logs with importance level of `info` or less to `combined.log`
     new transports.File({
       filename: `${logDir}/combined.log`,
-      format: format.combine(
-        format.json(),
-      ),
+      format: format.combine(format.json()),
     }),
     // Rotate logs to `application-%DATE%.log`
     new DailyRotateFile({
@@ -44,9 +71,7 @@ const logger = createLogger({
       zippedArchive: true,
       maxSize: '20m',
       utc: true,
-      format: format.combine(
-        format.json(),
-      ),
+      format: format.combine(format.json()),
     }),
   ],
   exceptionHandlers: [
@@ -73,22 +98,24 @@ const logger = createLogger({
       utc: true,
     }),
   ],
-  exitOnError: false
+  exitOnError: false,
 });
 
 //
 // If not in production then log to the `console`
 //
 if (process.env.NODE_ENV !== 'production') {
-  logger.add(new transports.Console({
-    format: format.combine(
-      format.colorize(),
-      format.prettyPrint(),
-      logFormat,
-    )
-  }));
+  logger.add(
+    new transports.Console({
+      format: format.combine(
+        format.colorize(),
+        format.prettyPrint(),
+        logFormat,
+      ),
+    }),
+  );
 }
 
 module.exports = {
   logger,
-};
+};
diff --git a/src/controllers/organization.controller.js b/src/controllers/organization.controller.js
@@ -9,6 +9,8 @@ import {
   assertCanDeleteOrg,
 } from '../utils/data-assertions';
 
+import { getDataModelVersion } from '../utils/helpers';
+
 import { ModelKeys, Audit, Staging } from '../models';
 
 export const findAll = async (req, res) => {
@@ -35,10 +37,15 @@ export const createV2 = async (req, res) => {
       const { name } = req.body;
       const buffer = req.files.file.data;
       const icon = `data:image/png;base64, ${buffer.toString('base64')}`;
+      const dataModelVersion = getDataModelVersion();
 
       return res.json({
         message: 'New organization created successfully.',
-        orgId: await Organization.createHomeOrganization(name, icon, 'v1'),
+        orgId: await Organization.createHomeOrganization(
+          name,
+          icon,
+          dataModelVersion,
+        ),
       });
     }
   } catch (error) {
@@ -64,9 +71,15 @@ export const create = async (req, res) => {
       });
     } else {
       const { name, icon } = req.body;
+      const dataModelVersion = getDataModelVersion();
+
       return res.json({
         message: 'New organization created successfully.',
-        orgId: await Organization.createHomeOrganization(name, icon, 'v1'),
+        orgId: await Organization.createHomeOrganization(
+          name,
+          icon,
+          dataModelVersion,
+        ),
       });
     }
   } catch (error) {
@@ -245,8 +258,7 @@ export const resyncOrganization = async (req, res) => {
     await transaction.commit();
 
     return res.json({
-      message:
-        'Resyncing organization completed',
+      message: 'Resyncing organization completed',
     });
   } catch (error) {
     res.status(400).json({

diff --git a/src/models/governance/governance.model.js b/src/models/governance/governance.model.js
@@ -8,28 +8,83 @@ import datalayer from '../../datalayer';
 import { keyValueToChangeList } from '../../utils/datalayer-utils';
 import { getConfig } from '../../utils/config-loader';
 import { logger } from '../../config/logger.cjs';
+import { getDataModelVersion } from '../../utils/helpers';
 
 const { GOVERANCE_BODY_ID, GOVERNANCE_BODY_IP, GOVERNANCE_BODY_PORT } =
   getConfig().GOVERNANCE;
 
+const { USE_SIMULATOR } = getConfig().APP;
+
 import ModelTypes from './governance.modeltypes.cjs';
 
 class Governance extends Model {
   static async createGoveranceBody() {
-    const goveranceBodyId = await datalayer.createDataLayerStore();
-
     if (GOVERANCE_BODY_ID && GOVERANCE_BODY_ID !== '') {
       throw new Error(
         'You are already listening to another governance body. Please clear GOVERANCE_BODY_ID from your env and try again',
       );
     }
 
-    await Meta.upsert({
-      metaKey: 'goveranceBodyId',
-      metaValue: goveranceBodyId,
-    });
+    const dataModelVersion = getDataModelVersion();
+    const goveranceBodyId = await datalayer.createDataLayerStore();
+    const governanceVersionId = await datalayer.createDataLayerVersion;
+
+    const revertOrganizationIfFailed = async () => {
+      logger.info('Reverting Failed Governance Body Creation');
+      await Meta.destroy({ where: { metaKey: 'goveranceBodyId' } });
+    };
+
+    // sync the governance store
+    await datalayer.syncDataLayer(
+      goveranceBodyId,
+      {
+        [dataModelVersion]: governanceVersionId,
+      },
+      revertOrganizationIfFailed,
+    );
 
-    return goveranceBodyId;
+    const onConfirm = () => {
+      logger.info('Organization confirmed, you are ready to go');
+      Meta.upsert({
+        metaKey: 'goveranceBodyId',
+        metaValue: governanceVersionId,
+      });
+    };
+
+    if (!USE_SIMULATOR) {
+      logger.info('Waiting for New Governance Body to be confirmed');
+      datalayer.getStoreData(
+        governanceVersionId,
+        onConfirm,
+        revertOrganizationIfFailed,
+      );
+    } else {
+      onConfirm();
+    }
+
+    return governanceVersionId;
+  }
+
+  static async upsertGovernanceDownload(governanceData) {
+    const updates = [];
+
+    if (governanceData.orgList) {
+      updates.push({
+        metaKey: 'orgList',
+        metaValue: governanceData.orgList,
+        confirmed: true,
+      });
+    }
+
+    if (governanceData.pickList) {
+      updates.push({
+        metaKey: 'pickList',
+        metaValue: governanceData.pickList,
+        confirmed: true,
+      });
+    }
+
+    await Promise.all(updates.map(async (update) => Governance.upsert(update)));
   }
 
   static async sync() {
@@ -44,32 +99,30 @@ class Governance extends Model {
         GOVERNANCE_BODY_PORT,
       );
 
-      logger.info('!!!');
-      logger.info(JSON.stringify(governanceData));
-
-      const updates = [];
-
-      if (governanceData.orgList) {
-        updates.push({
-          metaKey: 'orgList',
-          metaValue: governanceData.orgList,
-          confirmed: true,
-        });
-      }
+      // Check if there is v1, v2, v3 ..... and if not, then we assume this is a legacy goverance table that isnt versioned
+      const shouldSyncLegacy = !Object.keys(governanceData).some((key) =>
+        /^v?[0-9]+$/.test(key),
+      );
 
-      if (governanceData.pickList) {
-        updates.push({
-          metaKey: 'pickList',
-          metaValue: governanceData.pickList,
-          confirmed: true,
-        });
+      if (shouldSyncLegacy) {
+        await Governance.upsertGovernanceDownload(governanceData);
       }
 
-      logger.info(JSON.stringify(updates));
+      // Check if the governance data for this version exists
+      const dataModelVersion = getDataModelVersion();
+      if (governanceData[dataModelVersion]) {
+        const versionedGovernanceData = await datalayer.getSubscribedStoreData(
+          governanceData[dataModelVersion],
+          GOVERNANCE_BODY_IP,
+          GOVERNANCE_BODY_PORT,
+        );
 
-      await Promise.all(
-        updates.map(async (update) => Governance.upsert(update)),
-      );
+        await Governance.upsertGovernanceDownload(versionedGovernanceData);
+      } else {
+        throw new Error(
+          `Governance data is not available from this source for ${dataModelVersion} data model.`,
+        );
+      }
     } catch (error) {
       logger.error('Error Syncing Governance Data', error);
     }

diff --git a/src/utils/config-loader.js b/src/utils/config-loader.js
@@ -6,17 +6,22 @@ import path from 'path';
 
 import { logger } from '../config/logger.cjs';
 
+import { getDataModelVersion } from './helpers';
 import defaultConfig from './defaultConfig.json';
 
 export const getConfig = _.memoize(() => {
   const homeDir = os.homedir();
-  const configFile = path.resolve(
-    `${homeDir}/.chia/climate-warehouse/config.yaml`,
-  );
+  const dataModelVersion = getDataModelVersion();
+  const persistanceFolder = `${homeDir}/.chia/climate-warehouse/${dataModelVersion}`;
+  const configFile = path.resolve(`${persistanceFolder}/config.yaml`);
 
   try {
     if (!fs.existsSync(configFile)) {
       try {
+        if (!fs.existsSync(persistanceFolder)) {
+          fs.mkdirSync(persistanceFolder, { recursive: true });
+        }
+
         fs.writeFileSync(configFile, yaml.dump(defaultConfig), 'utf8');
       } catch (err) {
         // if it still doesnt exist that means we are in an env without write permissions