Merge pull request #485 from Chia-Network/feat/config-yaml

feat: load config from yaml

Dockerfile

@@ -3,7 +3,6 @@ FROM node:16
 WORKDIR /usr/src/app
 RUN npm install -g json
 COPY package*.json ./
-COPY .env.copy ./.env
 RUN json -I -f package.json -e "this.type=\"commonjs\""
 RUN npm set-script prepare ""
 RUN npm set-script requirements-check ""

package.json

@@ -19,7 +19,7 @@
     "prepare": "husky install",
     "build": "babel src --keep-file-extension --out-dir build --copy-files",
     "build-migrations": "babel migrations --keep-file-extension --out-dir dist/migrations --copy-files",
-    "prepare-binary": "rm -rf dist && mkdir dist && cp .env.copy dist/.env",
+    "prepare-binary": "rm -rf dist && mkdir dist",
     "create-win-x64-dist": "npm run build && npm run prepare-binary && pkg package.json -t node16-win-x64 --out-path dist",
     "create-mac-x64-dist": "npm run build && npm run prepare-binary && pkg package.json -t node16-macos-x64 --out-path dist",
     "create-linux-x64-dist": "npm run build && npm run prepare-binary && pkg package.json -t node16-linux-x64 --out-path dist"
@@ -37,6 +37,7 @@
     "express-fileupload": "^1.2.1",
     "express-joi-validation": "^5.0.0",
     "joi": "^17.5.0",
+    "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
     "log-update": "^4.0.0",
     "mysql2": "^2.3.3",

src/config/config.cjs

@@ -1,10 +1,51 @@
 require('dotenv').config();
 
+const _ = require('lodash');
+const yaml = require('js-yaml');
+const fs = require('fs');
 const os = require('os');
+const path = require('path');
 const homeDir = os.homedir();
+const defaultConfig = require('../utils/defaultConfig.json');
 
 const persistanceFolder = `${homeDir}/.chia/climate-warehouse`;
 
+// Adding this duplicate function here because im having trouble
+// importing it in from utils folder
+const getConfig = _.memoize(() => {
+  const configFile = path.resolve(
+    `${homeDir}/.chia/climate-warehouse/config.yaml`,
+  );
+
+  // First write it to chia home
+  if (!fs.existsSync(configFile)) {
+    try {
+      fs.writeFileSync(configFile, yaml.dump(defaultConfig), 'utf8');
+    } catch (err) {
+      // if it still doesnt exist that means we are in an env without write permissions
+      // so just load the default en
+      if (typeof process.env.USE_SIMULATOR === 'string') {
+        defaultConfig.APP.USE_SIMULATOR = process.env.USE_SIMULATOR === 'true';
+      }
+
+      console.log('Cant write config file, falling back to defaults');
+      return yaml.load(yaml.dump(defaultConfig));
+    }
+  }
+
+  try {
+    const yml = yaml.load(fs.readFileSync(configFile, 'utf8'));
+
+    if (typeof process.env.USE_SIMULATOR === 'string') {
+      yml.APP.USE_SIMULATOR = process.env.USE_SIMULATOR === 'true';
+    }
+
+    return yml;
+  } catch (e) {
+    console.log(e, `Config file not found at ${configFile}`);
+  }
+});
+
 module.exports = {
   local: {
     dialect: 'sqlite',
@@ -27,10 +68,10 @@ module.exports = {
     logging: false,
   },
   mirror: {
-    username: process.env.DB_USERNAME || '',
-    password: process.env.DB_PASSWORD || '',
-    database: process.env.DB_NAME || '',
-    host: process.env.DB_HOST || '',
+    username: getConfig().MIRROR_DB.DB_USERNAME || '',
+    password: getConfig().MIRROR_DB.DB_PASSWORD || '',
+    database: getConfig().MIRROR_DB.DB_NAME || '',
+    host: getConfig().MIRROR_DB.DB_HOST || '',
     dialect: 'mysql',
     logging: false,
   },

src/database/index.js

@@ -4,6 +4,7 @@ import Debug from 'debug';
 Debug.enable('climate-warehouse:mirror-database');
 const log = Debug('climate-warehouse:mirror-database');
 import mysql from 'mysql2/promise';
+import { getConfig } from '../utils/config-loader';
 
 import { migrations } from './migrations';
 import { seeders } from './seeders';
@@ -26,7 +27,10 @@ export const safeMirrorDbHandler = (callback) => {
         callback();
       })
       .catch(() => {
-        if (process.env.DB_HOST && process.env.DB_HOST !== '') {
+        if (
+          getConfig().MIRROR_DB.DB_HOST &&
+          getConfig().MIRROR_DB.DB_HOST !== ''
+        ) {
           log('Mirror DB not connected');
         }
       });
@@ -99,18 +103,18 @@ export const prepareDb = async () => {
 
   if (
     mirrorConfig == 'mirror' &&
-    process.env.DB_HOST &&
-    process.env.DB_HOST !== ''
+    getConfig().MIRROR_DB.DB_HOST &&
+    getConfig().MIRROR_DB.DB_HOST !== ''
   ) {
     const connection = await mysql.createConnection({
-      host: process.env.DB_HOST,
+      host: getConfig().MIRROR_DB.DB_HOST,
       port: 3306,
-      user: process.env.DB_USERNAME,
-      password: process.env.DB_PASSWORD,
+      user: getConfig().MIRROR_DB.DB_USERNAME,
+      password: getConfig().MIRROR_DB.DB_PASSWORD,
     });
 
     await connection.query(
-      `CREATE DATABASE IF NOT EXISTS \`${process.env.DB_NAME}\`;`,
+      `CREATE DATABASE IF NOT EXISTS \`${getConfig().MIRROR_DB.DB_NAME}\`;`,
     );
 
     const db = new Sequelize(config[mirrorConfig]);

src/datalayer/persistance.js

@@ -4,6 +4,7 @@ import fs from 'fs';
 import path from 'path';
 import request from 'request-promise';
 import os from 'os';
+import { getConfig } from '../utils/config-loader';
 
 import Debug from 'debug';
 Debug.enable('climate-warehouse:datalayer:persistance');
@@ -12,7 +13,7 @@ const log = Debug('climate-warehouse:datalayer:persistance');
 
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
-const rpcUrl = process.env.DATALAYER_URL;
+const rpcUrl = getConfig().APP.DATALAYER_URL;
 
 const getBaseOptions = () => {
   const homeDir = os.homedir();

src/datalayer/syncService.js

@@ -2,6 +2,7 @@ import _ from 'lodash';
 
 import { decodeHex, decodeDataLayerResponse } from '../utils/datalayer-utils';
 import { Organization, Staging, ModelKeys } from '../models';
+import { getConfig } from '../utils/config-loader';
 
 import * as dataLayer from './persistance';
 import * as simulator from './simulator';
@@ -10,6 +11,8 @@ import Debug from 'debug';
 Debug.enable('climate-warehouse:datalayer:syncService');
 const log = Debug('climate-warehouse:datalayer:syncService');
 
+const { USE_SIMULATOR } = getConfig().APP;
+
 const POLLING_INTERVAL = 5000;
 const frames = ['-', '\\', '|', '/'];
 
@@ -36,7 +39,7 @@ const startDataLayerUpdatePolling = async () => {
 const syncDataLayerStoreToClimateWarehouse = async (storeId, rootHash) => {
   let storeData;
 
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     storeData = await simulator.getStoreData(storeId, rootHash);
   } else {
     storeData = await dataLayer.getStoreData(storeId, rootHash);
@@ -119,7 +122,7 @@ const dataLayerWasUpdated = async () => {
   }
 
   let rootResponse;
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     rootResponse = await simulator.getRoots(subscribedOrgIds);
   } else {
     rootResponse = await dataLayer.getRoots(subscribedOrgIds);
@@ -160,7 +163,7 @@ const dataLayerWasUpdated = async () => {
 };
 
 const subscribeToStoreOnDataLayer = async (storeId, ip, port) => {
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     return simulator.subscribeToStoreOnDataLayer(storeId, ip, port);
   } else {
     return dataLayer.subscribeToStoreOnDataLayer(storeId, ip, port);
@@ -192,7 +195,7 @@ const getSubscribedStoreData = async (
     }
   }
 
-  if (process.env.USE_SIMULATOR !== 'true') {
+  if (!USE_SIMULATOR) {
     const storeExistAndIsConfirmed = await dataLayer.getRoot(storeId, true);
     if (!storeExistAndIsConfirmed) {
       log(`Retrying...`, retry + 1);
@@ -205,7 +208,7 @@ const getSubscribedStoreData = async (
   }
 
   let encodedData;
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     encodedData = await simulator.getStoreData(storeId);
   } else {
     encodedData = await dataLayer.getStoreData(storeId);
@@ -231,13 +234,13 @@ const getSubscribedStoreData = async (
 };
 
 const getRootHistory = (storeId) => {
-  if (process.env.USE_SIMULATOR !== 'true') {
+  if (!USE_SIMULATOR) {
     return dataLayer.getRootHistory(storeId);
   }
 };
 
 const getRootDiff = (storeId, root1, root2) => {
-  if (process.env.USE_SIMULATOR !== 'true') {
+  if (!USE_SIMULATOR) {
     return dataLayer.getRootDiff(storeId, root1, root2);
   }
 };

src/datalayer/wallet.js

@@ -2,10 +2,11 @@ import fs from 'fs';
 import path from 'path';
 import request from 'request-promise';
 import os from 'os';
+import { getConfig } from '../utils/config-loader';
 
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
-const rpcUrl = process.env.WALLET_URL;
+const rpcUrl = getConfig().APP.WALLET_URL;
 
 const getBaseOptions = () => {
   const homeDir = os.homedir();
@@ -81,7 +82,7 @@ const hasUnconfirmedTransactions = async () => {
 };
 
 const getPublicAddress = async () => {
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (getConfig().APP.USE_SIMULATOR) {
     return Promise.resolve('xch33300ddsje98f33hkkdf9dfuSIMULATED_ADDRESS');
   }
 

src/datalayer/writeService.js

@@ -4,14 +4,17 @@ import * as dataLayer from './persistance';
 import wallet from './wallet';
 import * as simulator from './simulator';
 import { encodeHex } from '../utils/datalayer-utils';
+import { getConfig } from '../utils/config-loader';
 
 import Debug from 'debug';
 Debug.enable('climate-warehouse:datalayer:writeService');
 const log = Debug('climate-warehouse:datalayer:writeService');
 
+const { USE_SIMULATOR } = getConfig().APP;
+
 const createDataLayerStore = async () => {
   let storeId;
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     storeId = await simulator.createDataLayerStore();
   } else {
     storeId = await dataLayer.createDataLayerStore();
@@ -58,7 +61,7 @@ const pushChangesWhenStoreIsAvailable = async (
   failedCallback = _.noop,
   retryAttempts = 0,
 ) => {
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     return simulator.pushChangeListToDataLayer(storeId, changeList);
   } else {
     const hasUnconfirmedTransactions =
@@ -87,7 +90,7 @@ const pushDataLayerChangeList = (storeId, changeList, failedCallback) => {
 };
 
 const dataLayerAvailable = async () => {
-  if (process.env.USE_SIMULATOR === 'true') {
+  if (USE_SIMULATOR) {
     return simulator.dataLayerAvailable();
   } else {
     return dataLayer.dataLayerAvailable();

src/models/governance/governance.model.js

@@ -6,14 +6,18 @@ import { sequelize } from '../../database';
 import { Meta } from '../../models';
 import datalayer from '../../datalayer';
 import { keyValueToChangeList } from '../../utils/datalayer-utils';
+import { getConfig } from '../../utils/config-loader';
+
+const { GOVERANCE_BODY_ID, GOVERNANCE_BODY_IP, GOVERNANCE_BODY_PORT } =
+  getConfig().GOVERNANCE;
 
 import ModelTypes from './governance.modeltypes.cjs';
 
 class Governance extends Model {
   static async createGoveranceBody() {
     const goveranceBodyId = await datalayer.createDataLayerStore();
 
-    if (process.env.GOVERANCE_BODY_ID && process.env.GOVERANCE_BODY_ID !== '') {
+    if (GOVERANCE_BODY_ID && GOVERANCE_BODY_ID !== '') {
       throw new Error(
         'You are already listening to another governance body. Please clear GOVERANCE_BODY_ID from your env and try again',
       );
@@ -28,9 +32,6 @@ class Governance extends Model {
   }
 
   static async sync() {
-    const { GOVERANCE_BODY_ID, GOVERNANCE_BODY_IP, GOVERNANCE_BODY_PORT } =
-      process.env;
-
     if (!GOVERANCE_BODY_ID || !GOVERNANCE_BODY_IP || !GOVERNANCE_BODY_PORT) {
       throw new Error('Missing information in env to sync Governance data');
     }