From 6e7043b7696744be61c1cfc5e774d9f3f2765e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Fri, 20 Sep 2024 17:36:00 +0000 Subject: [PATCH] refactor(log-db)!: Remove support for getting logs from Sense log db Implements #860 --- src/butler-sos.js | 6 - src/config/production_template.yaml | 18 --- src/globals.js | 40 +----- src/lib/config-file-schema.js | 2 - src/lib/config-obfuscate.js | 23 ---- src/lib/logdb.js | 183 ---------------------------- src/lib/post-to-mqtt.js | 8 -- src/lib/telemetry.js | 7 -- 8 files changed, 1 insertion(+), 286 deletions(-) delete mode 100755 src/lib/logdb.js diff --git a/src/butler-sos.js b/src/butler-sos.js index 56028fca..c335df22 100755 --- a/src/butler-sos.js +++ b/src/butler-sos.js @@ -14,7 +14,6 @@ promFastifyMetricsServer.register(metricsPlugin, { endpoint: '/metrics' }); // Load code from sub modules import { setupHealthMetricsTimer } from './lib/healthmetrics.js'; -import { setupLogDbTimer } from './lib/logdb.js'; import { setupUserSessionsTimer } from './lib/proxysessionmetrics.js'; import { setupAppNamesExtractTimer } from './lib/appnamesextract.js'; import { setupHeartbeatTimer } from './lib/heartbeat.js'; @@ -262,11 +261,6 @@ async function mainScript() { } } - // Set up extraction of data from log db - if (globals.config.get('Butler-SOS.logdb.enable') === true) { - setupLogDbTimer(); - } - // Set up extraction of sessions data if (globals.config.get('Butler-SOS.userSessions.enableSessionExtract') === true) { setupUserSessionsTimer(); diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index e3862b5b..3ed9bc04 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -352,22 +352,6 @@ Butler-SOS: error: true # Should error level log events be handled by Butler SOS? warn: true # Should warning level log events be handled by Butler SOS? - # Qlik Sense logging db config parameters - logdb: - enable: false - # Items below are mandatory if logdb.enable=true - pollingInterval: 60000 # How often (milliseconds) should Postgres log db be queried for warnings and errors? - queryPeriod: 5 minutes # How far back should Butler SOS query for log entries? - host: # E.g. 10.5.23.7 or sense.mycompany.com - port: 4432 # 4432 if using default Sense setup - qlogsReaderUser: qlogs_reader - qlogsReaderPwd: - extractErrors: true # Should error level entries be extracted from log db into Influxdb? - extractWarnings: true # Should warn level entries be extracted from log db into Influxdb? - extractInfo: false # Should info level entries be extracted from log db into Influxdb? - # Warning! Seting this to true will result in LOTS of log messages - # being retrrieved by Butler SOS! - # Certificates to use when connecting to Sense. Get these from the Certificate Export in QMC. cert: clientCert: @@ -549,7 +533,6 @@ Butler-SOS: - host: :4747 # Example: 10.34.3.45:4747 serverName: serverDescription: - logDbHost: userSessions: enable: true # Items below are mandatory if userSessions.enable=true @@ -569,7 +552,6 @@ Butler-SOS: - host: :4747 # Example: 10.34.3.46:4747 serverName: serverDescription: - logDbHost: userSessions: enable: true # Items below are mandatory if userSessions.enable=true diff --git a/src/globals.js b/src/globals.js index 59d473ba..b4a57055 100755 --- a/src/globals.js +++ b/src/globals.js @@ -352,28 +352,6 @@ class Settings { // Get info on what servers to monitor this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers'); - // Only set up connection pool for accessing Qlik Sense log db if that feature is enabled - this.pgPool; - if (this.config.get('Butler-SOS.logdb.enable') === true) { - const { Pool } = pg; - - // Set up connection pool for accessing Qlik Sense log db - this.pgPool = new Pool({ - host: this.config.get('Butler-SOS.logdb.host'), - database: 'QLogs', - user: this.config.get('Butler-SOS.logdb.qlogsReaderUser'), - password: this.config.get('Butler-SOS.logdb.qlogsReaderPwd'), - port: this.config.get('Butler-SOS.logdb.port'), - }); - - // the pool will emit an error on behalf of any idle clients - // it contains if a backend error or network partition happens - this.pgPool.on('error', (err, client) => { - this.logger.error(`CONFIG: Unexpected error on idle client: ${err}`); - // process.exit(-1); - }); - } - // Get list of standard and user configurable tags // ..begin with standard tags const tagValues = ['host', 'server_name', 'server_description']; @@ -461,11 +439,6 @@ class Settings { } } - // Create InfluxDB tags for data coming from log db - const tagValuesLogEventLogDb = tagValues.slice(); - tagValuesLogEventLogDb.push('source_process'); - tagValuesLogEventLogDb.push('log_level'); - // Create tags for user sessions const tagValuesUserProxySessions = tagValues.slice(); tagValuesUserProxySessions.push('user_session_virtual_proxy'); @@ -602,13 +575,6 @@ class Settings { }, tags: tagValues, }, - { - measurement: 'log_event_logdb', - fields: { - message: Influx.FieldType.STRING, - }, - tags: tagValuesLogEventLogDb, - }, { measurement: 'log_event', fields: { @@ -961,11 +927,7 @@ class Settings { (item) => item.iface === defaultNetworkInterface ); - const idSrc = - networkInterface[0].mac + - networkInterface[0].ip4 + - this.config.get('Butler-SOS.logdb.host') + - siSystem.uuid; + const idSrc = networkInterface[0].mac + networkInterface[0].ip4 + siSystem.uuid; const salt = networkInterface[0].mac; const hash = crypto.createHmac('sha256', salt); hash.update(idSrc); diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 6b5ab1f3..b56fd51a 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -1296,7 +1296,6 @@ export const confifgFileSchema = { host: { type: 'string' }, serverName: { type: 'string' }, serverDescription: { type: 'string' }, - logDbHost: { type: 'string' }, userSessions: { type: 'object', properties: { @@ -1335,7 +1334,6 @@ export const confifgFileSchema = { 'host', 'serverName', 'serverDescription', - 'logDbHost', 'userSessions', 'serverTags', 'headers', diff --git a/src/lib/config-obfuscate.js b/src/lib/config-obfuscate.js index 3827a5f2..f8c2ebc4 100644 --- a/src/lib/config-obfuscate.js +++ b/src/lib/config-obfuscate.js @@ -90,21 +90,6 @@ function configObfuscate(config) { obfuscatedConfig['Butler-SOS'].logEvents.sendToMQTT.baseTopic.substring(0, 10) + '*'.repeat(10); - // Log db - may not be present in the config in future versions of Butler SOS - if (obfuscatedConfig['Butler-SOS'].logdb) { - // Obfuscate Butler-SOS.logdb.host, keep first 3 chars, mask the rest with * - obfuscatedConfig['Butler-SOS'].logdb.host = - obfuscatedConfig['Butler-SOS'].logdb.host.substring(0, 3) + '*'.repeat(10); - - // Obfuscate Butler-SOS.logdb.qlogsReaderUser, keep first 3 chars, mask the rest with * - obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderUser = - obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderUser.substring(0, 3) + - '*'.repeat(10); - - // Obfuscate Butler-SOS.logdb.qlogsReaderPwdd, keep first 0 chars, mask the rest with * - obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderPwdd = '*'.repeat(10); - } - // Obfuscate Butler-SOS.cert.clientCert, keep first 10 chars, mask the rest with * obfuscatedConfig['Butler-SOS'].cert.clientCert = obfuscatedConfig['Butler-SOS'].cert.clientCert.substring(0, 10) + '*'.repeat(10); @@ -165,14 +150,6 @@ function configObfuscate(config) { host: element.host.substring(0, 3) + '*'.repeat(10), })); - // Obfuscate Butler-SOS.serversToMonitor.servers[].logDbHost, keep first 3 chars, mask the rest with * - obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig[ - 'Butler-SOS' - ].serversToMonitor.servers?.map((element) => ({ - ...element, - logDbHost: element.logDbHost.substring(0, 3) + '*'.repeat(10), - })); - // Obfuscate Butler-SOS.serversToMonitor.servers[].userSessions.host, keep first 3 chars, mask the rest with * obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig[ 'Butler-SOS' diff --git a/src/lib/logdb.js b/src/lib/logdb.js deleted file mode 100755 index 95826464..00000000 --- a/src/lib/logdb.js +++ /dev/null @@ -1,183 +0,0 @@ -/* eslint-disable no-unused-vars */ - -import globals from '../globals.js'; -import { postLogDbToMQTT } from './post-to-mqtt.js'; - -export function setupLogDbTimer() { - // Get query period from config file. - const queryPeriod = globals.config.get('Butler-SOS.logdb.queryPeriod'); - - // Configure timer for getting log data from Postgres - setInterval(() => { - globals.logger.verbose('LOGDB: Event started: Query log db'); - - // Create list of logging levels to include in query - const arrayincludeLogLevels = []; - if (globals.config.get('Butler-SOS.logdb.extractErrors')) { - arrayincludeLogLevels.push("'ERROR'"); - } - if (globals.config.get('Butler-SOS.logdb.extractWarnings')) { - arrayincludeLogLevels.push("'WARN'"); - } - if (globals.config.get('Butler-SOS.logdb.extractInfo')) { - arrayincludeLogLevels.push("'INFO'"); - } - const includeLogLevels = arrayincludeLogLevels.join(); - - // checkout a Postgres client from connection pool - globals.pgPool - .connect() - .then((pgClient) => - pgClient - .query( - `select - id, - entry_timestamp as timestamp, - entry_level, - process_host, - process_name, - payload - from public.log_entries - where - entry_level in (${includeLogLevels}) and - (entry_timestamp > now() - INTERVAL '${queryPeriod}' ) - order by - entry_timestamp desc - ` - ) - .then((res) => { - pgClient.release(); - globals.logger.debug('LOGDB: Got query response.'); - - const { rows } = res; - rows.forEach((row) => { - globals.logger.silly(`LOGDB: Row: ${JSON.stringify(row)}`); - - // Post to Influxdb (if enabled) - if (globals.config.get('Butler-SOS.influxdbConfig.enable') === true) { - globals.logger.silly(`LOGDB: Posting log db data to Influxdb...`); - - // Make sure that the payload message exists - storing it to Influx would otherwise throw an error - // if (!row.payload.hasOwnProperty('Message')) { - // Suggested by GitHub Copilot: - if (!Object.prototype.hasOwnProperty.call(row.payload, 'Message')) { - // eslint-disable-next-line no-param-reassign - row.payload.Message = ''; - } - - // Get all tags for the current server. - // Some special logic is needed to match the host value returned from Postgres with the logDbHost property from - // the YAML config file. - // Once we have that match we can add all the tags for that server. - const serverItem = globals.serverList.find((item) => { - globals.logger.silly( - `LOGDB: Matching logdb host "${row.process_host}" against config file logDbHost "${item.logDbHost}"` - ); - return item.logDbHost === row.process_host; - }); - - // NOTE: If no match is found above, i.e. serverItem == undefined, this means that a Sense host name was returned, - // but no server in the YAML config file had a matching logDbHost setting. - // This is an error and should be sent to the log. - // Also, only store data into Influxdb for servers defined in YAML config file. - - if (serverItem === undefined) { - globals.logger.verbose( - `LOGDB: No logDbHost config file entries matching host name received from Sense: ${row.process_host}` - ); - globals.logger.verbose( - `LOGDB: Hint: Consider adding "${row.process_host}" as a logDbHost entry in the config file...` - ); - } else { - // group = serverItem.serverTags.serverGroup; - const srvName = serverItem.serverName; - const srvDesc = serverItem.serverDescription; - - let tagsForDbEntry = { - host: row.process_host, - server_name: srvName, - server_description: srvDesc, - source_process: row.process_name, - log_level: row.entry_level, - }; - - // Add all tags defined for this server in the config file - if ( - // serverItem.hasOwnProperty('serverTags') && - // Suggestions by GitHub Copilot: - Object.prototype.hasOwnProperty.call( - serverItem, - 'serverTags' - ) && - serverItem.serverTags !== null - ) { - // Loop over all tags defined for the current server, adding them to the data structure that will later be passed to Influxdb - Object.entries(serverItem.serverTags).forEach((entry) => { - tagsForDbEntry = Object.assign(tagsForDbEntry, { - [entry[0]]: entry[1], - }); - }); - - globals.logger.silly( - `LOGDB: Tags passed to Influxdb as part of logdb record: ${JSON.stringify( - tagsForDbEntry - )}` - ); - } - - // Write the whole reading to Influxdb - globals.influx - .writePoints([ - { - measurement: 'log_event_logdb', - tags: tagsForDbEntry, - fields: { - message: row.payload.Message, - }, - timestamp: row.timestamp, - }, - ]) - .then((_err) => { - globals.logger.silly( - 'LOGDB: Sent log db event to Influxdb' - ); - }) - .catch((err) => { - globals.logger.error( - `LOGDB: Error saving log event to InfluxDB! ${err.stack}` - ); - globals.logger.error( - `LOGDB: Full error: ${JSON.stringify(err)}` - ); - }); - } - - // Post to MQTT (if enabled) - if (globals.config.get('Butler-SOS.mqttConfig.enable') === true) { - globals.logger.silly('LOGDB: Posting log db data to MQTT...'); - postLogDbToMQTT( - row.process_host, - row.process_name, - row.entry_level, - row.payload.Message, - row.timestamp - ); - } - } - }); - }) - .then((_res) => { - globals.logger.verbose('LOGDB: Sent log event to Influxdb'); - }) - .catch((err) => { - globals.logger.error(`LOGDB: Log db query error: ${err.stack}`); - // pgClient.release(); - }) - ) - .catch((err) => { - globals.logger.error( - `LOGDB: ERROR: Could not connect to Postgres log db: ${err.stack}` - ); - }); - }, globals.config.get('Butler-SOS.logdb.pollingInterval')); -} diff --git a/src/lib/post-to-mqtt.js b/src/lib/post-to-mqtt.js index f8739b53..5b325f85 100755 --- a/src/lib/post-to-mqtt.js +++ b/src/lib/post-to-mqtt.js @@ -3,14 +3,6 @@ import globals from '../globals.js'; -export function postLogDbToMQTT(processHost, processName, entryLevel, message, _timestamp) { - // Get base MQTT topic - const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic'); - - // Send to MQTT - globals.mqttClient.publish(`${baseTopic + processHost}/${processName}/${entryLevel}`, message); -} - export function postHealthToMQTT(_host, serverName, body) { // Get base MQTT topic const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic'); diff --git a/src/lib/telemetry.js b/src/lib/telemetry.js index 25363c2c..7d0b57ae 100644 --- a/src/lib/telemetry.js +++ b/src/lib/telemetry.js @@ -29,7 +29,6 @@ const callRemoteURL = async function reportTelemetry() { let logEventsMQTTEnable = false; let logEventsInfluxDBEnable = false; let logEventsNewRelicEnable = false; - let logdbEnable = false; let mqttEnable = false; let newRelicEnable = false; let prometheusEnable = false; @@ -137,10 +136,6 @@ const callRemoteURL = async function reportTelemetry() { logEventEnginePerformanceMonitorTrackRejectedEnable = true; } - if (globals.config.get('Butler-SOS.logdb.enable') === true) { - logdbEnable = true; - } - if (globals.config.get('Butler-SOS.mqttConfig.enable') === true) { mqttEnable = true; } @@ -214,7 +209,6 @@ const callRemoteURL = async function reportTelemetry() { feature_logEventsInfluxdb: logEventsInfluxDBEnable, feature_logEventsNewRelic: logEventsNewRelicEnable, - feature_logdb: logdbEnable, feature_mqtt: mqttEnable, feature_newRelic: newRelicEnable, feature_prometheus: prometheusEnable, @@ -263,7 +257,6 @@ const callRemoteURL = async function reportTelemetry() { logEventsMQTT: logEventsMQTTEnable, logEventsInfluxdb: logEventsInfluxDBEnable, logEventsNewRelic: logEventsNewRelicEnable, - logdb: logdbEnable, mqtt: mqttEnable, newRelic: newRelicEnable, prometheus: prometheusEnable,