feat(qs-events): Add counters for incoming Qlik Sense events

Implements #884

package.json

@@ -36,6 +36,7 @@
         "@fastify/static": "^7.0.4",
         "@influxdata/influxdb-client": "^1.35.0",
         "@influxdata/influxdb-client-apis": "^1.35.0",
+        "async-mutex": "^0.5.0",
         "axios": "^1.7.5",
         "commander": "^12.1.0",
         "config": "^3.3.12",

src/butler-sos.js

@@ -25,6 +25,7 @@ import { setupAnonUsageReportTimer } from './lib/telemetry.js';
 import { setupPromClient } from './lib/prom-client.js';
 import { verifyConfigFile } from './lib/config-file-verify.js';
 import { setupConfigVisServer } from './lib/config-visualise.js';
+import { setupUdpEventsStorage } from './lib/udp-event.js';
 
 // Suppress experimental warnings
 // https://stackoverflow.com/questions/55778283/how-to-disable-warnings-when-node-is-launched-via-a-global-shell-script
@@ -51,7 +52,6 @@ process.emit = function (name, data, ...args) {
 };
 
 async function sleep(ms) {
-    // eslint-disable-next-line no-promise-executor-return
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
@@ -81,7 +81,6 @@ async function mainScript() {
     // Sleep 5 seconds otherwise to llow globals to be initialised
 
     function sleepLocal(ms) {
-        // eslint-disable-next-line no-promise-executor-return
         return new Promise((resolve) => setTimeout(resolve, ms));
     }
 
@@ -285,6 +284,12 @@ async function mainScript() {
     if (globals.config.get('Butler-SOS.configVisualisation.enable') === true) {
         await setupConfigVisServer();
     }
+
+    // Set up rejected user/log events storage, if enabled
+    if (globals.config.get('Butler-SOS.qlikSenseEvents.rejectedEventCount.enable') === true) {
+        globals.logger.verbose('MAIN: Rejected events storage enabled');
+        await setupUdpEventsStorage();
+    }
 }
 
 mainScript();

src/config/production_template.yaml

@@ -77,6 +77,27 @@ Butler-SOS:
       #   insertApiKey: <API key 2 (with insert permissions) from New Relic> 
       #   accountId: <New Relic account ID 2>
 
+  # Shared settings for user and log events (see below)
+  qlikSenseEvents:                  # Shared settings for user and log events (see below)
+    influxdb:
+      enable: true                  # Should summary (counter) of user and log events be stored in InfluxDB?
+      writeFrequency: 20000         # How often (milliseconds) should rejected event count be written to InfluxDB?  
+    eventCount:                     # Track how many events are received from Sense.
+                                    # Some events are valid, some are not. Of the valid events, some are rejected by Butler SOS
+                                    # based on the configuration in this file. 
+      enable: true                  # Should event count be stored in InfluxDB?
+      influxdb:
+        measurementName: event_count # Name of the InfluxDB measurement where event count is stored
+        tags:                       # Tags are added to the data before it's stored in InfluxDB
+          # - tag: env
+          #   value: DEV
+          # - tag: foo
+          #   value: bar
+    rejectedEventCount:
+      enable: true                  # Should rejected events be counted and stored in InfluxDB?
+      influxdb:
+        measurementName: rejected_event_count # Name of the InfluxDB measurement where rejected event count is stored
+
   # Track individual users opening/closing apps and starting/stopping sessions. 
   # Requires log appender XML file(s) to be added to Sense server(s).
   userEvents:                       
@@ -205,10 +226,17 @@ Butler-SOS:
         category:
           - name: qs_log_category
             value: unknown
-    appPerformanceMonitor:              # Detailed app performance data extraction from log events
-      enable: false                      # Should app performance data be extracted from log events?
+    enginePerformanceMonitor:           # Detailed app performance data extraction from log events
+      enable: false                     # Should app performance data be extracted from log events?
       appNameLookup:                    # Should app names be looked up based on app IDs?
-        enable: true
+        enable: false
+      trackRejectedEvents: 
+        enable: false                   # Should events that are rejected by the app performance monitor be tracked?
+        tags:                           # Tags are added to the data before it's stored in InfluxDB
+          # - tag: env
+          #   value: DEV
+          # - tag: foo
+          #   value: bar
       monitorFilter:                    # What objects should be monitored? Entire apps or just specific object(s) within some specific app(s)?
                                         # Two kinds of monitoring can be done:
                                         # 1) Monitor all apps, except those listed for exclusion. This is defined in the allApps section.

src/globals.js

@@ -13,9 +13,10 @@ import Influx from 'influx';
 import { Command, Option } from 'commander';
 import { InfluxDB, HttpError, DEFAULT_WriteOptions } from '@influxdata/influxdb-client';
 import { OrgsAPI, BucketsAPI } from '@influxdata/influxdb-client-apis';
+import { fileURLToPath } from 'url';
 
 import { getServerTags } from './lib/servertags.js';
-import { fileURLToPath } from 'url';
+import { UdpEvents } from './lib/udp-event.js';
 
 let instance = null;
 
@@ -122,7 +123,6 @@ class Settings {
             configFileBasename = upath.basename(this.configFile, configFileExtension);
 
             if (configFileExtension.toLowerCase() !== '.yaml') {
-                // eslint-disable-next-line no-console
                 console.log('Error: Config file extension must be yaml');
                 process.exit(1);
             }
@@ -131,7 +131,6 @@ class Settings {
                 process.env.NODE_CONFIG_DIR = configFilePath;
                 process.env.NODE_ENV = configFileBasename;
             } else {
-                // eslint-disable-next-line no-console
                 console.log('Error: Specified config file does not exist');
                 process.exit(1);
             }
@@ -192,24 +191,19 @@ class Settings {
 
             // Are we in a packaged app?
             if (this.isPkg) {
-                // eslint-disable-next-line no-console
                 console.log(`Running in packaged app. Executable path: ${this.execPath}`);
             } else {
-                // eslint-disable-next-line no-console
                 console.log(
                     `Running in non-packaged environment. Executable path: ${this.execPath}`
                 );
             }
 
-            // eslint-disable-next-line no-console
             console.log(
                 `Log file directory: ${upath.join(this.execPath, this.config.get('Butler-SOS.logDirectory'))}`
             );
 
-            // eslint-disable-next-line no-console
             console.log(`upath.dirname(process.execPath): ${upath.dirname(process.execPath)}`);
 
-            // eslint-disable-next-line no-console
             console.log(`process.cwd(): ${process.cwd()}`);
         }
 
@@ -338,6 +332,22 @@ class Settings {
             this.logger.error(`CONFIG: Setting up UDP log events listener: ${err}`);
         }
 
+        // ------------------------------------
+        // Track user events and log events
+        if (this.config.get('Butler-SOS.qlikSenseEvents.eventCount.enable') === true) {
+            this.udpEvents = new UdpEvents(this.logger);
+        } else {
+            this.udpEvents = null;
+        }
+
+        // ------------------------------------
+        // Track rejected user and log events
+        if (this.config.get('Butler-SOS.qlikSenseEvents.rejectedEventCount.enable') === true) {
+            this.rejectedEvents = new UdpEvents(this.logger);
+        } else {
+            this.rejectedEvents = null;
+        }
+
         // ------------------------------------
         // Get info on what servers to monitor
         this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers');
@@ -358,7 +368,6 @@ class Settings {
 
             // the pool will emit an error on behalf of any idle clients
             // it contains if a backend error or network partition happens
-            // eslint-disable-next-line no-unused-vars
             this.pgPool.on('error', (err, client) => {
                 this.logger.error(`CONFIG: Unexpected error on idle client: ${err}`);
                 // process.exit(-1);
@@ -710,7 +719,6 @@ class Settings {
 
     // Static sleep function
     static sleep(ms) {
-        // eslint-disable-next-line no-promise-executor-return
         return new Promise((resolve) => setTimeout(resolve, ms));
     }
 

src/lib/config-file-schema.js

@@ -65,6 +65,30 @@ export const confifgFileSchema = {
                 },
             ],
         },
+
+        qlikSenseEvents: {
+            influxdb: {
+                enable: 'boolean',
+                writeFrequency: 'number',
+            },
+            eventCount: {
+                enable: 'boolean',
+                influxdb: {
+                    measurementName: 'string',
+                    "tags?": [
+                        "tag": 'string',
+                        value: 'string',
+                    ],
+                },
+            },
+            rejectedEventCount: {
+                enable: 'boolean',
+                influxdb: {
+                    measurementName: 'string',
+                },
+            },
+        },
+
         userEvents: {
             enable: 'boolean',
             'excludeUser?': [
@@ -173,11 +197,20 @@ export const confifgFileSchema = {
                     ],
                 },
             },
-            appPerformanceMonitor: {
+            enginePerformanceMonitor: {
                 enable: 'boolean',
                 appNameLookup: {
                     enable: 'boolean',
                 },
+                trackRejectedEvents: {
+                    enable: 'boolean',
+                    "tags?": [
+                        {
+                            tag: 'string',
+                            value: 'string',
+                        },
+                    ],
+                },
                 monitorFilter: {
                     allApps: {
                         enable: 'boolean',