Merge pull request #904 from mountaindude/860

860
ptarmiganlabs · Sep 20, 2024 · dd275c7 · dd275c7
2 parents 9adecbb + 7336fb7
commit dd275c7
Show file tree

Hide file tree

Showing 12 changed files with 1,537 additions and 3,912 deletions.
diff --git a/docs/docker-compose/docker-compose.yml b/docs/docker-compose/docker-compose.yml
@@ -1,14 +1,14 @@
 # docker-compose.yml
-version: "3.3"
 services:
     butler-sos:
         image: ptarmiganlabs/butler-sos:latest
         container_name: butler-sos
         restart: always
         ports:
-            - "9997:9997"
-            - "9996:9996"
-            - "9842:9842"
+            - "9997:9997"   # UDP user events
+            - "9996:9996"   # UDP log events
+            - "9842:9842"   # Prometheus metrics
+            - "3100:3100"   # Config file visualization
         volumes:
             # Make config file accessible outside of container
             - "./config:/nodeapp/config"

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -36,6 +36,8 @@
         "@fastify/static": "^7.0.4",
         "@influxdata/influxdb-client": "^1.35.0",
         "@influxdata/influxdb-client-apis": "^1.35.0",
+        "ajv": "^8.17.1",
+        "ajv-keywords": "^5.1.0",
         "async-mutex": "^0.5.0",
         "axios": "^1.7.7",
         "commander": "^12.1.0",
@@ -50,16 +52,15 @@
         "lodash.clonedeep": "^4.5.0",
         "luxon": "^3.5.0",
         "mqtt": "^5.10.1",
-        "pg": "^8.12.0",
+        "pg": "^8.13.0",
         "posthog-node": "^4.2.0",
         "prom-client": "^15.1.3",
         "qrs-interact": "^6.3.1",
         "systeminformation": "^5.23.5",
-        "ua-parser-js": "^1.0.38",
+        "ua-parser-js": "^1.0.39",
         "uuid": "^10.0.0",
         "winston": "^3.14.2",
-        "winston-daily-rotate-file": "^5.0.0",
-        "yaml-validator": "^5.0.1"
+        "winston-daily-rotate-file": "^5.0.0"
     },
     "devDependencies": {
         "@eslint/js": "^9.10.0",

diff --git a/src/butler-sos.js b/src/butler-sos.js
@@ -14,7 +14,6 @@ promFastifyMetricsServer.register(metricsPlugin, { endpoint: '/metrics' });
 
 // Load code from sub modules
 import { setupHealthMetricsTimer } from './lib/healthmetrics.js';
-import { setupLogDbTimer } from './lib/logdb.js';
 import { setupUserSessionsTimer } from './lib/proxysessionmetrics.js';
 import { setupAppNamesExtractTimer } from './lib/appnamesextract.js';
 import { setupHeartbeatTimer } from './lib/heartbeat.js';
@@ -262,11 +261,6 @@ async function mainScript() {
         }
     }
 
-    // Set up extraction of data from log db
-    if (globals.config.get('Butler-SOS.logdb.enable') === true) {
-        setupLogDbTimer();
-    }
-
     // Set up extraction of sessions data
     if (globals.config.get('Butler-SOS.userSessions.enableSessionExtract') === true) {
         setupUserSessionsTimer();

diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml
@@ -352,22 +352,6 @@ Butler-SOS:
             error: true                # Should error level log events be handled by Butler SOS?
             warn: true                 # Should warning level log events be handled by Butler SOS?
 
-  # Qlik Sense logging db config parameters
-  logdb:
-    enable: false
-    # Items below are mandatory if logdb.enable=true
-    pollingInterval: 60000            # How often (milliseconds) should Postgres log db be queried for warnings and errors?
-    queryPeriod: 5 minutes            # How far back should Butler SOS query for log entries?
-    host: <IP or FQDN of Qlik Sense logging db>   # E.g. 10.5.23.7 or sense.mycompany.com
-    port: 4432                        # 4432 if using default Sense setup
-    qlogsReaderUser: qlogs_reader
-    qlogsReaderPwd: <pwd>
-    extractErrors: true               # Should error level entries be extracted from log db into Influxdb?
-    extractWarnings: true             # Should warn level entries be extracted from log db into Influxdb?
-    extractInfo: false                # Should info level entries be extracted from log db into Influxdb? 
-                                      # Warning! Seting this to true will result in LOTS of log messages 
-                                      # being retrrieved by Butler SOS!
-
   # Certificates to use when connecting to Sense. Get these from the Certificate Export in QMC.
   cert:
     clientCert: <path/to/cert/client.pem>
@@ -549,7 +533,6 @@ Butler-SOS:
       - host: <server1.my.domain>:4747        # Example: 10.34.3.45:4747
         serverName: <server1>
         serverDescription: <description>
-        logDbHost: <host name as used in QLogs db>
         userSessions:
           enable: true
           # Items below are mandatory if userSessions.enable=true
@@ -569,7 +552,6 @@ Butler-SOS:
       - host: <server2.my.domain>:4747        # Example: 10.34.3.46:4747
         serverName: <server2>
         serverDescription: <description>
-        logDbHost: <host name as used in QLogs db>
         userSessions:
           enable: true
           # Items below are mandatory if userSessions.enable=true

diff --git a/src/globals.js b/src/globals.js
@@ -352,28 +352,6 @@ class Settings {
         // Get info on what servers to monitor
         this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers');
 
-        // Only set up connection pool for accessing Qlik Sense log db if that feature is enabled
-        this.pgPool;
-        if (this.config.get('Butler-SOS.logdb.enable') === true) {
-            const { Pool } = pg;
-
-            // Set up connection pool for accessing Qlik Sense log db
-            this.pgPool = new Pool({
-                host: this.config.get('Butler-SOS.logdb.host'),
-                database: 'QLogs',
-                user: this.config.get('Butler-SOS.logdb.qlogsReaderUser'),
-                password: this.config.get('Butler-SOS.logdb.qlogsReaderPwd'),
-                port: this.config.get('Butler-SOS.logdb.port'),
-            });
-
-            // the pool will emit an error on behalf of any idle clients
-            // it contains if a backend error or network partition happens
-            this.pgPool.on('error', (err, client) => {
-                this.logger.error(`CONFIG: Unexpected error on idle client: ${err}`);
-                // process.exit(-1);
-            });
-        }
-
         // Get list of standard and user configurable tags
         // ..begin with standard tags
         const tagValues = ['host', 'server_name', 'server_description'];
@@ -461,11 +439,6 @@ class Settings {
             }
         }
 
-        // Create InfluxDB tags for data coming from log db
-        const tagValuesLogEventLogDb = tagValues.slice();
-        tagValuesLogEventLogDb.push('source_process');
-        tagValuesLogEventLogDb.push('log_level');
-
         // Create tags for user sessions
         const tagValuesUserProxySessions = tagValues.slice();
         tagValuesUserProxySessions.push('user_session_virtual_proxy');
@@ -602,13 +575,6 @@ class Settings {
                             },
                             tags: tagValues,
                         },
-                        {
-                            measurement: 'log_event_logdb',
-                            fields: {
-                                message: Influx.FieldType.STRING,
-                            },
-                            tags: tagValuesLogEventLogDb,
-                        },
                         {
                             measurement: 'log_event',
                             fields: {
@@ -961,11 +927,7 @@ class Settings {
                 (item) => item.iface === defaultNetworkInterface
             );
 
-            const idSrc =
-                networkInterface[0].mac +
-                networkInterface[0].ip4 +
-                this.config.get('Butler-SOS.logdb.host') +
-                siSystem.uuid;
+            const idSrc = networkInterface[0].mac + networkInterface[0].ip4 + siSystem.uuid;
             const salt = networkInterface[0].mac;
             const hash = crypto.createHmac('sha256', salt);
             hash.update(idSrc);