don't clearDatastore

.github/workflows/buildTestPublishContainerDeploy.yml

@@ -3,7 +3,6 @@ on:
   push:
     branches:
       - main
-      - issue48
   release:
     types:
       - published

docs/docs/config/cronJobs.md

@@ -8,6 +8,8 @@ _NotifyBC_ runs several cron jobs described below. These jobs are controlled by
 
 By default cron jobs are enabled. In a multi-node deployment, cron jobs should only run on the [master node](../config-nodeRoles/) to ensure single execution.
 
+All cron jobs share common config item <a name="timeSpec"></a>_timeSpec_ - a space separated fields conformed to [unix crontab format](<https://www.freebsd.org/cgi/man.cgi?crontab(5)>) with an optional left-most seconds field. See [allowed ranges](https://github.com/kelektiv/node-cron#cron-ranges) of each field.
+
 ## Purge Data
 
 This cron job purges old notifications, subscriptions and notification bounces. The default frequency of cron job and retention policy are defined by _cron.purgeData_ config object in file _/src/config.ts_
@@ -16,6 +18,7 @@ This cron job purges old notifications, subscriptions and notification bounces.
 module.exports = {
   cron: {
     purgeData: {
+      // daily at 1am
       timeSpec: '0 0 1 * * *',
       pushNotificationRetentionDays: 30,
       expiredInAppNotificationRetentionDays: 30,
@@ -28,9 +31,8 @@ module.exports = {
 };
 ```
 
-The config items are
+where
 
-- <a name="timeSpec"></a>_timeSpec_: a space separated fields conformed to [unix crontab format](<https://www.freebsd.org/cgi/man.cgi?crontab(5)>) with an optional left-most seconds field. See [allowed ranges](https://github.com/kelektiv/node-cron#cron-ranges) of each field.
 - _pushNotificationRetentionDays_: the retention days of push notifications
 - _expiredInAppNotificationRetentionDays_: the retention days of expired inApp notifications
 - _nonConfirmedSubscriptionRetentionDays_: the retention days of non-confirmed subscriptions, i.e. all unconfirmed and deleted subscriptions
@@ -58,14 +60,13 @@ This cron job sends out future-dated notifications when the notification becomes
 module.exports = {
   cron: {
     dispatchLiveNotifications: {
+      // minutely
       timeSpec: '0 * * * * *',
     },
   },
 };
 ```
 
-_timeSpec_ follows [same syntax described above](#timeSpec).
-
 ## Check Rss Config Updates
 
 This cron job monitors RSS feed notification dynamic config items. If a config item is created, updated or deleted, the cron job starts, restarts, or stops the RSS-specific cron job. The default config is defined by _cron.checkRssConfigUpdates_ config object in file _/src/config.ts_
@@ -74,13 +75,14 @@ This cron job monitors RSS feed notification dynamic config items. If a config i
 module.exports = {
   cron: {
     checkRssConfigUpdates: {
+      // minutely
       timeSpec: '0 * * * * *',
     },
   },
 };
 ```
 
-_timeSpec_ follows [same syntax described above](#timeSpec). Note this _timeSpec_ doesn't control the RSS poll frequency (which is defined in dynamic configs and is service specific), instead it only controls the frequency to check for dynamic config changes.
+Note _timeSpec_ doesn't control the RSS poll frequency (which is defined in dynamic configs and is service specific), instead it only controls the frequency to check for dynamic config changes.
 
 ## Delete Notification Bounces
 
@@ -95,6 +97,7 @@ The default config is defined by _cron.deleteBounces_ config object in file _/sr
 module.exports = {
   cron: {
     deleteBounces: {
+      // hourly
       timeSpec: '0 0 * * * *',
       minLapsedHoursSinceLatestNotificationEnded: 1,
     },
@@ -104,5 +107,38 @@ module.exports = {
 
 where
 
-- _timeSpec_ is the frequency of cron job, following [same syntax described above](#timeSpec)
 - _minLapsedHoursSinceLatestNotificationEnded_ is the time span
+
+## Re-dispatch Broadcast Push Notifications
+
+This cron job re-dispatches a broadcast push notifications when original request failed. It is part of [guaranteed broadcast push dispatch processing](../config/notification.md#guaranteed-broadcast-push-dispatch-processing)
+
+The default config is defined by _cron.reDispatchBroadcastPushNotifications_ config object in file _/src/config.ts_
+
+```ts
+module.exports = {
+  cron: {
+    reDispatchBroadcastPushNotifications: {
+      // minutely
+      timeSpec: '0 * * * * *',
+    },
+  },
+};
+```
+
+## Clear Redis Datastore
+
+This cron job clears Redis datastore, including updating settings, used for SMS [throttle](../config/sms.md#throttle). The job is enabled only if Redis is used. Datastore is cleared only when there is no SMS broadcast push notifications in _sending_ state. Without this cron job, updated throttle settings in config file will never take effect, and staled jobs in Redis datastore will not be cleaned up.
+
+The default config is defined by _cron.clearRedisDatastore_ config object in file _/src/config.ts_
+
+```ts
+module.exports = {
+  cron: {
+    clearRedisDatastore: {
+      // hourly
+      timeSpec: '0 0 * * * *',
+    },
+  },
+};
+```

src/__tests__/acceptance/cron.acceptance.ts

@@ -646,3 +646,40 @@ describe('CRON reDispatchBroadcastPushNotifications', function () {
     expect(data[0].dispatch?.successful).containEql('1');
   });
 });
+
+describe('CRON clearRedisDatastore', function () {
+  let ready: sinon.SinonStub;
+  let disconnect: sinon.SinonStub;
+  beforeEach(async function () {
+    ready = sinon.stub().resolves(null);
+    disconnect = sinon.stub().resolves(null);
+    sinon.stub(cronTasks, 'Bottleneck').returns({
+      ready,
+      disconnect,
+    });
+  });
+  it('should not clear datastore when there is sending notification', async function () {
+    await notificationRepository.create({
+      channel: 'sms',
+      message: {
+        textBody: 'this is a test http://foo.com',
+      },
+      isBroadcast: true,
+      serviceName: 'myService',
+      state: 'sending',
+    });
+    await cronTasks.clearRedisDatastore(app)();
+    sinon.assert.notCalled(cronTasks.Bottleneck);
+    sinon.assert.notCalled(ready);
+    sinon.assert.notCalled(disconnect);
+  });
+
+  it('should clear datastore when no sending notification', async function () {
+    await cronTasks.clearRedisDatastore(app)();
+    sinon.assert.calledOnceWithMatch(cronTasks.Bottleneck, {
+      clearDatastore: true,
+    });
+    sinon.assert.calledOnce(ready);
+    sinon.assert.calledOnce(disconnect);
+  });
+});

src/config.ts

@@ -70,9 +70,8 @@ const config: AnyObject = {
     throttle: {
       id: 'notifyBCSms',
       minTime: 250,
-      clearDatastore: true,
       /* Redis clustering options */
-      // datastore: 'redis',
+      // datastore: 'ioredis',
       // clientOptions: {
       //   host: '127.0.0.1',
       //   port: 6379,
@@ -146,7 +145,7 @@ const config: AnyObject = {
   },
   cron: {
     purgeData: {
-      // daily
+      // daily at 1am
       timeSpec: '0 0 1 * * *',
       pushNotificationRetentionDays: 30,
       expiredInAppNotificationRetentionDays: 30,
@@ -172,6 +171,10 @@ const config: AnyObject = {
       // minutely
       timeSpec: '0 * * * * *',
     },
+    clearRedisDatastore: {
+      // hourly
+      timeSpec: '0 0 * * * *',
+    },
   },
   tls: {},
 };

src/observers/cron-tasks.ts

@@ -14,6 +14,7 @@
 
 import {Application, CoreBindings} from '@loopback/core';
 import {AnyObject} from '@loopback/repository';
+import Bottleneck from 'bottleneck';
 import {Rss, RssItem, RssRelations} from '../models';
 import {
   AccessTokenRepository,
@@ -29,6 +30,7 @@ const request = require('axios');
 const _ = require('lodash');
 
 module.exports.request = request;
+module.exports.Bottleneck = Bottleneck;
 module.exports.purgeData = async (app: Application) => {
   const cronConfig: AnyObject =
     (await app.getConfig(
@@ -570,3 +572,31 @@ module.exports.reDispatchBroadcastPushNotifications = (app: Application) => {
     );
   };
 };
+
+module.exports.clearRedisDatastore = (app: Application) => {
+  return async () => {
+    const notificationRepository: NotificationRepository = await app.get(
+      'repositories.NotificationRepository',
+    );
+    const sendingSmsNotification = await notificationRepository.findOne(
+      {
+        where: {
+          state: 'sending',
+          channel: 'sms',
+          isBroadcast: true,
+        },
+      },
+      undefined,
+    );
+    if (sendingSmsNotification) return;
+    const smsThrottleConfig = <Bottleneck.ConstructorOptions>(
+      await app.getConfig(CoreBindings.APPLICATION_INSTANCE, 'sms.throttle')
+    );
+    const newSmsThrottleConfig = Object.assign({}, smsThrottleConfig, {
+      clearDatastore: true,
+    });
+    const smsLimiter = new module.exports.Bottleneck(newSmsThrottleConfig);
+    await smsLimiter.ready();
+    await smsLimiter.disconnect();
+  };
+};

src/observers/cron.observer.ts

@@ -97,6 +97,26 @@ export class CronObserver implements LifeCycleObserver {
         start: true,
       }),
     );
+
+    const smsThrottleConfig: AnyObject =
+      (await this.app.getConfig(
+        CoreBindings.APPLICATION_INSTANCE,
+        'sms.throttle',
+      )) ?? {};
+    if (
+      smsThrottleConfig.clearDatastore !== true &&
+      smsThrottleConfig.datastore !== 'local'
+    ) {
+      const clearRedisDatastore = cronConfig.clearRedisDatastore || {};
+      this.jobs.push(
+        new CronJob({
+          cronTime: clearRedisDatastore.timeSpec,
+          onTick: cronTasks.clearRedisDatastore(this.app),
+          start: true,
+          runOnInit: true,
+        }),
+      );
+    }
   }
   async stop(): Promise<void> {
     for (const job of this.jobs) {