refactor(log-db)!: Remove support for getting logs from Sense log db

Implements #860
ptarmiganlabs · Sep 20, 2024 · 6e7043b · 6e7043b
1 parent 9be5d8a
commit 6e7043b
Show file tree

Hide file tree

Showing 8 changed files with 1 addition and 286 deletions.
diff --git a/src/butler-sos.js b/src/butler-sos.js
@@ -14,7 +14,6 @@ promFastifyMetricsServer.register(metricsPlugin, { endpoint: '/metrics' });
 
 // Load code from sub modules
 import { setupHealthMetricsTimer } from './lib/healthmetrics.js';
-import { setupLogDbTimer } from './lib/logdb.js';
 import { setupUserSessionsTimer } from './lib/proxysessionmetrics.js';
 import { setupAppNamesExtractTimer } from './lib/appnamesextract.js';
 import { setupHeartbeatTimer } from './lib/heartbeat.js';
@@ -262,11 +261,6 @@ async function mainScript() {
         }
     }
 
-    // Set up extraction of data from log db
-    if (globals.config.get('Butler-SOS.logdb.enable') === true) {
-        setupLogDbTimer();
-    }
-
     // Set up extraction of sessions data
     if (globals.config.get('Butler-SOS.userSessions.enableSessionExtract') === true) {
         setupUserSessionsTimer();

diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml
@@ -352,22 +352,6 @@ Butler-SOS:
             error: true                # Should error level log events be handled by Butler SOS?
             warn: true                 # Should warning level log events be handled by Butler SOS?
 
-  # Qlik Sense logging db config parameters
-  logdb:
-    enable: false
-    # Items below are mandatory if logdb.enable=true
-    pollingInterval: 60000            # How often (milliseconds) should Postgres log db be queried for warnings and errors?
-    queryPeriod: 5 minutes            # How far back should Butler SOS query for log entries?
-    host: <IP or FQDN of Qlik Sense logging db>   # E.g. 10.5.23.7 or sense.mycompany.com
-    port: 4432                        # 4432 if using default Sense setup
-    qlogsReaderUser: qlogs_reader
-    qlogsReaderPwd: <pwd>
-    extractErrors: true               # Should error level entries be extracted from log db into Influxdb?
-    extractWarnings: true             # Should warn level entries be extracted from log db into Influxdb?
-    extractInfo: false                # Should info level entries be extracted from log db into Influxdb? 
-                                      # Warning! Seting this to true will result in LOTS of log messages 
-                                      # being retrrieved by Butler SOS!
-
   # Certificates to use when connecting to Sense. Get these from the Certificate Export in QMC.
   cert:
     clientCert: <path/to/cert/client.pem>
@@ -549,7 +533,6 @@ Butler-SOS:
       - host: <server1.my.domain>:4747        # Example: 10.34.3.45:4747
         serverName: <server1>
         serverDescription: <description>
-        logDbHost: <host name as used in QLogs db>
         userSessions:
           enable: true
           # Items below are mandatory if userSessions.enable=true
@@ -569,7 +552,6 @@ Butler-SOS:
       - host: <server2.my.domain>:4747        # Example: 10.34.3.46:4747
         serverName: <server2>
         serverDescription: <description>
-        logDbHost: <host name as used in QLogs db>
         userSessions:
           enable: true
           # Items below are mandatory if userSessions.enable=true

diff --git a/src/globals.js b/src/globals.js
@@ -352,28 +352,6 @@ class Settings {
         // Get info on what servers to monitor
         this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers');
 
-        // Only set up connection pool for accessing Qlik Sense log db if that feature is enabled
-        this.pgPool;
-        if (this.config.get('Butler-SOS.logdb.enable') === true) {
-            const { Pool } = pg;
-
-            // Set up connection pool for accessing Qlik Sense log db
-            this.pgPool = new Pool({
-                host: this.config.get('Butler-SOS.logdb.host'),
-                database: 'QLogs',
-                user: this.config.get('Butler-SOS.logdb.qlogsReaderUser'),
-                password: this.config.get('Butler-SOS.logdb.qlogsReaderPwd'),
-                port: this.config.get('Butler-SOS.logdb.port'),
-            });
-
-            // the pool will emit an error on behalf of any idle clients
-            // it contains if a backend error or network partition happens
-            this.pgPool.on('error', (err, client) => {
-                this.logger.error(`CONFIG: Unexpected error on idle client: ${err}`);
-                // process.exit(-1);
-            });
-        }
-
         // Get list of standard and user configurable tags
         // ..begin with standard tags
         const tagValues = ['host', 'server_name', 'server_description'];
@@ -461,11 +439,6 @@ class Settings {
             }
         }
 
-        // Create InfluxDB tags for data coming from log db
-        const tagValuesLogEventLogDb = tagValues.slice();
-        tagValuesLogEventLogDb.push('source_process');
-        tagValuesLogEventLogDb.push('log_level');
-
         // Create tags for user sessions
         const tagValuesUserProxySessions = tagValues.slice();
         tagValuesUserProxySessions.push('user_session_virtual_proxy');
@@ -602,13 +575,6 @@ class Settings {
                             },
                             tags: tagValues,
                         },
-                        {
-                            measurement: 'log_event_logdb',
-                            fields: {
-                                message: Influx.FieldType.STRING,
-                            },
-                            tags: tagValuesLogEventLogDb,
-                        },
                         {
                             measurement: 'log_event',
                             fields: {
@@ -961,11 +927,7 @@ class Settings {
                 (item) => item.iface === defaultNetworkInterface
             );
 
-            const idSrc =
-                networkInterface[0].mac +
-                networkInterface[0].ip4 +
-                this.config.get('Butler-SOS.logdb.host') +
-                siSystem.uuid;
+            const idSrc = networkInterface[0].mac + networkInterface[0].ip4 + siSystem.uuid;
             const salt = networkInterface[0].mac;
             const hash = crypto.createHmac('sha256', salt);
             hash.update(idSrc);

diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js
@@ -1296,7 +1296,6 @@ export const confifgFileSchema = {
                                     host: { type: 'string' },
                                     serverName: { type: 'string' },
                                     serverDescription: { type: 'string' },
-                                    logDbHost: { type: 'string' },
                                     userSessions: {
                                         type: 'object',
                                         properties: {
@@ -1335,7 +1334,6 @@ export const confifgFileSchema = {
                                     'host',
                                     'serverName',
                                     'serverDescription',
-                                    'logDbHost',
                                     'userSessions',
                                     'serverTags',
                                     'headers',

diff --git a/src/lib/config-obfuscate.js b/src/lib/config-obfuscate.js
@@ -90,21 +90,6 @@ function configObfuscate(config) {
             obfuscatedConfig['Butler-SOS'].logEvents.sendToMQTT.baseTopic.substring(0, 10) +
             '*'.repeat(10);
 
-        // Log db - may not be present in the config in future versions of Butler SOS
-        if (obfuscatedConfig['Butler-SOS'].logdb) {
-            // Obfuscate Butler-SOS.logdb.host, keep first 3 chars, mask the rest with *
-            obfuscatedConfig['Butler-SOS'].logdb.host =
-                obfuscatedConfig['Butler-SOS'].logdb.host.substring(0, 3) + '*'.repeat(10);
-
-            // Obfuscate Butler-SOS.logdb.qlogsReaderUser, keep first 3 chars, mask the rest with *
-            obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderUser =
-                obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderUser.substring(0, 3) +
-                '*'.repeat(10);
-
-            // Obfuscate Butler-SOS.logdb.qlogsReaderPwdd, keep first 0 chars, mask the rest with *
-            obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderPwdd = '*'.repeat(10);
-        }
-
         // Obfuscate Butler-SOS.cert.clientCert, keep first 10 chars, mask the rest with *
         obfuscatedConfig['Butler-SOS'].cert.clientCert =
             obfuscatedConfig['Butler-SOS'].cert.clientCert.substring(0, 10) + '*'.repeat(10);
@@ -165,14 +150,6 @@ function configObfuscate(config) {
             host: element.host.substring(0, 3) + '*'.repeat(10),
         }));
 
-        // Obfuscate Butler-SOS.serversToMonitor.servers[].logDbHost, keep first 3 chars, mask the rest with *
-        obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig[
-            'Butler-SOS'
-        ].serversToMonitor.servers?.map((element) => ({
-            ...element,
-            logDbHost: element.logDbHost.substring(0, 3) + '*'.repeat(10),
-        }));
-
         // Obfuscate Butler-SOS.serversToMonitor.servers[].userSessions.host, keep first 3 chars, mask the rest with *
         obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig[
             'Butler-SOS'

diff --git a/src/lib/logdb.js b/src/lib/logdb.js
diff --git a/src/lib/post-to-mqtt.js b/src/lib/post-to-mqtt.js
@@ -3,14 +3,6 @@
 
 import globals from '../globals.js';
 
-export function postLogDbToMQTT(processHost, processName, entryLevel, message, _timestamp) {
-    // Get base MQTT topic
-    const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic');
-
-    // Send to MQTT
-    globals.mqttClient.publish(`${baseTopic + processHost}/${processName}/${entryLevel}`, message);
-}
-
 export function postHealthToMQTT(_host, serverName, body) {
     // Get base MQTT topic
     const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic');