fix: improve mirror logic

.github/dependabot.yml

@@ -4,4 +4,5 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+      day: "sunday"
     target-branch: "develop"

.gitignore

@@ -1,6 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 data.sqlite3
 test.sqlite3
+test.sqlite3*
 testMirror.sqlite3
 
 # dependencies

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cadt",
-  "version": "1.7.10",
+  "version": "1.7.11",
   "_comment": "DONT CHANGE MAJOR UNLESS DATAMODEL CHANGES: The major version corresponds to the datamodel version your using, so 2.0.0 means it'll use datamodel v2",
   "private": true,
   "bin": "build/server.js",
@@ -60,18 +60,18 @@
   },
   "devDependencies": {
     "@babel/cli": "^7.23.9",
-    "@babel/core": "^7.23.9",
+    "@babel/core": "^7.24.0",
     "@babel/plugin-syntax-import-attributes": "^7.23.3",
-    "@babel/preset-env": "^7.23.9",
+    "@babel/preset-env": "^7.24.0",
     "@babel/register": "^7.23.7",
-    "@commitlint/cli": "^18.6.1",
-    "@commitlint/config-conventional": "^18.6.2",
+    "@commitlint/cli": "^19.0.3",
+    "@commitlint/config-conventional": "^19.0.3",
     "babel-plugin-module-resolver": "^5.0.0",
     "chai": "^5.1.0",
     "chai-http": "^4.4.0",
-    "eslint": "^8.56.0",
+    "eslint": "^8.57.0",
     "eslint-plugin-es": "^4.1.0",
-    "eslint-plugin-mocha": "^10.2.0",
+    "eslint-plugin-mocha": "^10.3.0",
     "husky": "^9.0.11",
     "mocha": "^10.3.0",
     "semver": "^7.6.0",

src/datalayer/persistance.js

@@ -3,12 +3,11 @@ import fs from 'fs';
 import path from 'path';
 import superagent from 'superagent';
 import { getConfig } from '../utils/config-loader';
-import fullNode from './fullNode';
-import { publicIpv4 } from '../utils/ip-tools';
 import wallet from './wallet';
 import { Organization } from '../models';
 import { logger } from '../config/logger.cjs';
 import { getChiaRoot } from '../utils/chia-root.js';
+import { getMirrorUrl } from '../utils/datalayer-utils';
 
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
@@ -117,6 +116,17 @@ const addMirror = async (storeId, url, forceAddMirror = false) => {
   await wallet.waitForAllTransactionsToConfirm();
   const homeOrg = await Organization.getHomeOrg();
 
+  logger.info(
+    `Checking mirrors for storeID is ${storeId} with mirror URL ${url}`,
+  );
+
+  if (!url) {
+    logger.info(
+      `No DATALAYER_FILE_SERVER_URL specified so skipping mirror for ${storeId}`,
+    );
+    return false;
+  }
+
   if (!homeOrg && !forceAddMirror) {
     logger.info(`No home org detected so skipping mirror for ${storeId}`);
     return false;
@@ -130,7 +140,7 @@ const addMirror = async (storeId, url, forceAddMirror = false) => {
   );
 
   if (mirror) {
-    logger.info(`Mirror already available for ${storeId}`);
+    logger.info(`Mirror already available for ${storeId} at ${url}`);
     return true;
   }
 
@@ -553,12 +563,9 @@ const subscribeToStoreOnDataLayer = async (storeId) => {
     if (Object.keys(data).includes('success') && data.success) {
       logger.info(`Successfully Subscribed: ${storeId}`);
 
-      const chiaConfig = fullNode.getChiaConfig();
+      const mirrorUrl = await getMirrorUrl();
 
-      await addMirror(
-        storeId,
-        `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`,
-      );
+      await addMirror(storeId, mirrorUrl, true);
 
       return data;
     }

src/datalayer/writeService.js

@@ -38,8 +38,8 @@ const createDataLayerStore = async () => {
   return storeId;
 };
 
-const addMirror = async (storeId, url) => {
-  return dataLayer.addMirror(storeId, url);
+const addMirror = async (storeId, url, force = false) => {
+  return dataLayer.addMirror(storeId, url, force);
 };
 
 const waitForStoreToBeConfirmed = async (storeId, retry = 0) => {

src/models/organizations/organizations.model.js

@@ -222,8 +222,8 @@ class Organization extends Model {
     return registryVersionId;
   }
 
-  static async addMirror(storeId, url) {
-    await datalayer.addMirror(storeId, url);
+  static async addMirror(storeId, url, force = false) {
+    await datalayer.addMirror(storeId, url, force);
   }
 
   static async importHomeOrg(orgUid) {

src/tasks/mirror-check.js

@@ -43,18 +43,19 @@ const job = new SimpleIntervalJob(
 );
 
 const runMirrorCheck = async () => {
-  const homeOrg = Organization.getHomeOrg();
-
-  if (homeOrg) {
-    const organizations = Organization.getOrgsMap();
-    const orgs = Object.keys(organizations);
-    for (const org of orgs) {
-      const orgData = organizations[org];
-      const mirrorUrl = await getMirrorUrl();
-
+  const organizations = await Organization.getOrgsMap();
+  const orgs = Object.keys(organizations);
+  for (const org of orgs) {
+    const orgData = organizations[org];
+    const mirrorUrl = await getMirrorUrl();
+    if (mirrorUrl) {
       // There is logic within the addMirror function to check if the mirror already exists
-      await Organization.addMirror(orgData.orgUid, mirrorUrl);
-      await Organization.addMirror(orgData.registryId, mirrorUrl);
+      await Organization.addMirror(orgData.orgUid, mirrorUrl, true);
+      await Organization.addMirror(orgData.registryId, mirrorUrl, true);
+    } else {
+      logger.error(
+        'DATALAYER_FILE_SERVER_URL not set, skipping mirror announcement',
+      );
     }
   }
 };

src/utils/datalayer-utils.js

@@ -1,6 +1,5 @@
 import { getConfig } from './config-loader';
-import fullNode from '../datalayer/fullNode';
-import { publicIpv4 } from './ip-tools';
+import { logger } from '../config/logger.cjs';
 
 export const encodeHex = (str) => {
   return Buffer.from(str).toString('hex');
@@ -113,10 +112,12 @@ export const optimizeAndSortKvDiff = (kvDiff) => {
 };
 
 export const getMirrorUrl = async () => {
-  const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
-  const chiaConfig = fullNode.getChiaConfig();
-  return (
-    DATALAYER_FILE_SERVER_URL ||
-    `http://${await publicIpv4()}:${chiaConfig.data_layer.host_port}`
-  );
+  try {
+    const { DATALAYER_FILE_SERVER_URL } = getConfig().APP;
+    logger.info(`Resolved Mirror Url: ${DATALAYER_FILE_SERVER_URL}`);
+    return DATALAYER_FILE_SERVER_URL;
+  } catch (error) {
+    logger.error('Error getting DATALAYER_FILE_SERVER_URL: ${error}');
+    return null;
+  }
 };