Merge pull request #870 from Chia-Network/develop

Release 1.6.1

.github/workflows/auto-release.yml

@@ -39,8 +39,33 @@ jobs:
             git add CHANGELOG.md
             git config --global user.name 'ChiaAutomation'
             git config --global user.email '[email protected]'
-            git commit -m "Updating changelog for $version"
+            git commit -m "chore: Updating changelog for $version"
             git tag $version
             git push origin $version
             git push origin main
           fi
+
+      - name: Checkout develop branch
+        uses: actions/checkout@v3
+        with:
+          # Need PACKAGE_ADMIN_PAT token so when the tag is created, the tag automation runs
+          token: ${{ secrets.PACKAGE_ADMIN_PAT }}
+          fetch-depth: 0
+          ref: 'develop'
+
+      - name: Auto-update dev dependencies
+        run: |
+          echo "Checking and updating the package.json file now..."
+          if $(npx --yes npm-check-updates --dep dev -u | grep -q "All dependencies match the latest package versions")
+          then
+            echo "No updates found - exiting"
+            exit 0
+          else
+            echo "Applying updates to the lock file..."
+            npm update -D
+            echo "Committing updates to the main branch"
+            git config --global user.name 'ChiaAutomation'
+            git config --global user.email '[email protected]'
+            git commit -m "chore: Updating npm dev dependencies"
+            git push origin develop
+          fi

.github/workflows/build-docker.yaml

@@ -31,7 +31,7 @@ jobs:
       - name: Setup Node 16.x
         uses: actions/setup-node@v3
         with:
-          node-version: '16.13'
+          node-version: '16.14'
 
       - name: Ignore Husky
         run: npm pkg delete scripts.prepare

.github/workflows/build.yaml

@@ -42,8 +42,8 @@ jobs:
       - name: Setup Node 16.x
         uses: actions/setup-node@v3
         with:
-          node-version: '16.13'
-      
+          node-version: '16.14'
+
       - name: Install Husky
         run: npm install --save-dev husky
 
@@ -181,7 +181,7 @@ jobs:
       - name: Build arm 64 dist
         run: |
           mkdir pkgcache
-          docker run --rm --platform linux/arm64 -v $(pwd):/app -w /app -e PKG_CACHE_PATH=pkgcache node:16.13 /bin/bash -c "npm pkg delete scripts.prepare && npm install && npm i -g @babel/cli @babel/preset-env pkg && npm run create-linux-arm64-dist"
+          docker run --rm --platform linux/arm64 -v $(pwd):/app -w /app -e PKG_CACHE_PATH=pkgcache node:16.14 /bin/bash -c "npm pkg delete scripts.prepare && npm install && npm i -g @babel/cli @babel/preset-env pkg && npm run create-linux-arm64-dist"
 
       - name: Copy sqlite3
         run: |

.github/workflows/tests.yaml

@@ -16,7 +16,7 @@ jobs:
     name: NPM Tests
     runs-on: ubuntu-latest
     container:
-      image: node:16.13
+      image: node:16.14
 
     steps:
       - uses: Chia-Network/actions/clean-workspace@main

.nvmrc

@@ -1 +1 @@
-16.0.0
+16.14

README.md

@@ -97,9 +97,17 @@ chmod ug+x .git/hooks/*
 npm run start
 ```
 
+### Ports, Networking, and Security
+
+By default, the CADT API will listen on localhost only on port 31310. If running a node with `READ_ONLY` set to `false`, it is highly recommended that CADT is run on a private network or with access limited by IP address. To allow remote connections to CADT, set the `BIND_ADDRESS` (see the [Configuration](#configuration) section below) to the IP to listen on, or `0.0.0.0` to listen on all interfaces. The port for the CADT API can be set with the parameter `CW_PORT`.  The default port is 31310. In many cases, users will need to access the API from their workstations for either the [CADT UI](https://github.com/Chia-Network/climate-warehouse-ui) or to integrate with existing tools and scripts.  To add authentication to the API, use the `CADT_API_KEY` parameter.  Alternatively, the API can be served behind an authentication proxy to restrict access and the `CADT_API_KEY` can be left blank.  If running an observer node with `READ_ONLY` set to `true`, the CADT API will only share data from the public blockchain, and running without authentication is usually safe.  If `READ_ONLY` is set to `false`, authentication must be used to prevent unauthorized writes to the blockchain. 
+
+### Adding Encryption to the CADT API
+
+The CADT API uses HTTP and is unencrypted.  To add encryption, use a reverse proxy like [Nginx](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) with an SSL certificate.  In this scenario, the CADT application can be set to listen only on localhost and Nginx (on the same server) would proxy incoming requests to port 31310.
+
 ### Configuration
 
-In the `CHIA_ROOT` directory (usually `~/.chia/mainnet` on Linux), CADT will add a directory called `climate-warehouse/v1` when the application is first run (in fact, this directory could be deleted at any time and CADT will recreate it next time it is started).  The main CADT configuration file is called `config.yaml` and can be found in this directory.  The options in this file are as follows (the full list of available options can be seen in the [config template](src/utils/defaultConfig.json)):
+In the `CHIA_ROOT` directory (usually `~/.chia/mainnet` on Linux), CADT will add a directory called `cadt/v1` when the application is first run (in fact, this directory could be deleted at any time and CADT will recreate it next time it is started).  The main CADT configuration file is called `config.yaml` and can be found in this directory.  The options in this file are as follows (the full list of available options can be seen in the [config template](src/utils/defaultConfig.json)):
 
 * **MIRROR_DB**: This section is for configuring the MySQL-compatible database that can be used for easy querying for report generation. This is optional and only provides a read-only mirror of the data CADT uses. 
   *  **DB_USERNAME**:  MySQL username
@@ -108,6 +116,7 @@ In the `CHIA_ROOT` directory (usually `~/.chia/mainnet` on Linux), CADT will add
   *  **DB_HOST**: Hostname of the MySQL database
 * **APP**:  This section is for configuring the CADT application.
   * **CW_PORT**: CADT port where the API will be available. 31310 by default.
+  * **BIND_ADDRESS**: By default, CADT listens on localhost only. To enable remote connections to CADT, change this to `0.0.0.0` to listen on all network interfaces, or to an IP address to listen on a specific network interface. 
   * **DATALAYER_URL**: URL and port to connect to the [Chia DataLayer RPC](https://docs.chia.net/datalayer-rpc).  If Chia is installed locally with default settings, https://localhost:8562 will work. 
   * **WALLET_URL**: URL and port to conned to the [Chia Wallet RPC](https://docs.chia.net/wallet-rpc).  If Chia is installed on the same machine as CADT with default settings, https://localhost:9256 will work.
   * **USE_SIMULATOR**: Developer setting to populate CADT from a governance file and enables some extra APIs.  Should always be "false" under normal usage. 
@@ -119,21 +128,18 @@ In the `CHIA_ROOT` directory (usually `~/.chia/mainnet` on Linux), CADT will add
   * **DEFAULT_FEE**: [Fee](https://docs.chia.net/mempool/) for each transaction on the Chia blockchain in mojos.  The default is 300000000 mojos (0.0003 XCH) and can be set higher or lower depending on how [busy](https://dashboard.chia.net/d/46EAA05E/mempool-transactions-and-fees?orgId=1) the Chia network is.  If a fee is set very low, it may cause a delay in transaction processing.  
   * **DEFAULT_COIN_AMOUNT**: Units are mojo.  Each DataLayer transaction needs a coin amount and the default is 300000000 mojo.  
   * **DATALAYER_FILE_SERVER_URL**: Chia DataLayer HTTP URL and port.  If serving DataLayer files from S3, this would be the public URL of the S3 bucket.  Must be publicly available.  
+  * **TASKS**: Section for configuring sync intervals
+    * **AUDIT_SYNC_TASK_INTERVAL**:  Default 30
+    * **DATAMODEL_SYNC_TASK_INTERVAL**:  Default 60
+    * **GOVERNANCE_SYNC_TASK_INTERVAL**:  Default 86400
+    * **ORGANIZATION_META_SYNC_TASK_INTERVAL**:  Default 86400
+    * **PICKLIST_SYNC_TASK_INTERVAL**:  Default 30
 * **GOVERNANCE**: Section on settings for the Governance body to connect to.
-  * **GOVERNANCE_BODY_ID**: This determines the governance body your CADT network will be connected to.  While there could be multiple governance body IDs, the default of 23f6498e015ebcd7190c97df30c032de8deb5c8934fc1caa928bc310e2b8a57e is the right ID for most people. 
-* **TASKS**: Section for configuring sync intervals
-  * **AUDIT_SYNC_TASK_INTERVAL**:  Default 30
-  * **DATAMODEL_SYNC_TASK_INTERVAL**:  Default 60
-  * **GOVERNANCE_SYNC_TASK_INTERVAL**:  Default 86400
-  * **ORGANIZATION_META_SYNC_TASK_INTERVAL**:  Default 86400
-  * **PICKLIST_SYNC_TASK_INTERVAL**:  Default 30
+  * **GOVERNANCE_BODY_ID**: This determines the governance body your CADT network will be connected to.  While there could be multiple governance body IDs, the default of `23f6498e015ebcd7190c97df30c032de8deb5c8934fc1caa928bc310e2b8a57e` is the right ID for most people. 
+
 ​
 Note that the CADT application will need to be restarted after any changes to the config.yaml file. 
 
-### Ports, Networking, and Security
-
-The port for the CADT API can be set with the parameter `CW_PORT` in the `config.yaml` file discussed above.  The default port is 31310.  The CADT API will listen on all network interfaces on this port so care must be taken to block this port at the firewall or networking level to avoid this API being public.  In many cases, the API will need to be public for either the [CADT UI](https://github.com/Chia-Network/climate-warehouse-ui) or to integrate with existing tools and scripts.  To add authentication to the API, use the `CADT_API_KEY` parameter.  Alternatively, the API can be served behind an authentication proxy to restrict access and the `CADT_API_KEY` can be left blank.  If running an observer node with `READ_ONLY` set to `true`, the CADT API will only share data from the public blockchain, and running without authentication is usually safe.  If `READ_ONLY` is set to `false`, authentication must be used to prevent unauthorized writes to the blockchain. 
-
 ## Developer Guide
 ​
 ### Build Binaries

check_node_version.js

@@ -1,5 +1,5 @@
 import semver from 'semver';
-import config from './package.json';
+import config from './package.json' assert { type: 'json' };
 const { engines } = config;
 
 const version = engines.node;

package.json

@@ -1,11 +1,12 @@
 {
   "name": "cadt",
-  "version": "1.6.0",
+  "version": "1.6.1",
+  "_comment": "DONT CHANGE MAJOR UNLESS DATAMODEL CHANGES: The major version corresponds to the datamodel version your using, so 2.0.0 means it'll use datamodel v2",
   "private": true,
   "bin": "build/server.js",
   "type": "module",
   "engines": {
-    "node": ">=16.0"
+    "node": ">=16.14"
   },
   "scripts": {
     "requirements-check": "node --experimental-json-modules check_node_version.js",

src/database/index.js

@@ -1,3 +1,4 @@
+import _ from 'lodash';
 import { Sequelize } from 'sequelize';
 import config from '../config/config.js';
 import { logger } from '../config/logger.cjs';
@@ -19,6 +20,11 @@ export const sequelizeMirror = new Sequelize(config[mirrorConfig]);
 
 logger.info('CADT:mirror-database');
 
+const logDebounce = _.debounce(() => {
+  console.log('Mirror DB not connected');
+  logger.info('Mirror DB not connected');
+}, 120000);
+
 export const safeMirrorDbHandler = (callback) => {
   try {
     sequelizeMirror
@@ -31,7 +37,7 @@ export const safeMirrorDbHandler = (callback) => {
         }
       })
       .catch(() => {
-        logger.info('Mirror DB not connected');
+        logDebounce();
       });
   } catch (error) {
     logger.error(