From 818c702a791f983506eb7d2c038e2f930b0b6299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Mon, 9 Sep 2024 11:59:43 +0000 Subject: [PATCH] feat(qix performance): Add fine-grained performance monitoring for app objects Implements #320 --- src/config/production_template.yaml | 82 +++++ src/globals.js | 1 - src/lib/config-file-schema.js | 63 ++++ src/lib/config-file-verify.js | 1 + src/lib/post-to-influxdb.js | 90 ++++-- src/lib/post-to-new-relic.js | 2 +- src/lib/udp_handlers_log_events.js | 473 +++++++++++++++++++++++++++- 7 files changed, 684 insertions(+), 28 deletions(-) diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 82e67130..4d61e47e 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -205,6 +205,88 @@ Butler-SOS: category: - name: qs_log_category value: unknown + appPerformanceMonitor: # Detailed app performance data extraction from log events + enable: false # Should app performance data be extracted from log events? + appNameLookup: # Should app names be looked up based on app IDs? + enable: true + monitorFilter: # What objects should be monitored? Entire apps or just specific object(s) within some specific app(s)? + # Two kinds of monitoring can be done: + # 1) Monitor all apps, except those listed for exclusion. This is defined in the allApps section. + # 2) Monitor only specific apps. This is defined in the appSpecific section. + # An event will be accepted if it matches any of the rules in the allApps section OR any of the rules in the appSpecific section. + allApps: + enable: false # Should all apps be monitored? + appExclude: # What apps should be excluded from monitoring? + # If both appId and appName are specified, both must match the event's data for it to be considered a match. + - appId: 5b817efe-472d-43ce-8a31-6cce34af7de9 + - appName: Sales forecast + - appId: f42d6b16-8faf-45ca-a783-59f9da47db6e + appName: Inventory analysis + objectType: + allObjectTypes: true # Should all object types be monitored? + allObjectTypesExclude: # If allObjectTypes is set to true, the object types in this array are excluded from monitoring. + # someObjectTypesInclude (below) is ignored in that case. + - LoadModelList + - + - linechart + - map + someObjectTypesInclude: # What object types should be included in monitoring? + # Only applicable if allObjectTypes is set to false. + - LoadModelList + - sheet + - barchart + method: + allMethods: true # Should all methods be monitored? + allMethodsExclude: # If allMethods is set to true, the methods in this array are excluded from monitoring. + # someMethodsInclude (below) is ignored in that case. + - Global::OpenApp + - Doc::GetAppLayout + - Doc::CreateSessionObject + someMethodsInclude: # What methods should be included in monitoring? + # Only applicable if allMethods is set to false. + - GenericObject::GetLayout + - GenericObject::GetHyperCubeContinuousData + appSpecific: + enable: false # Should app specific monitoring be done? + app: + - include: # What apps should be monitored? + # If both appId and appName are specified, both must match the event's data for it to be considered a match. + - appId: d7cf16f9-6a95-462a-9ff1-a6d413326de4 + - appName: Budget 2025 + - appId: 6931136d-c234-4358-a40c-e37153aba7c9 + appName: Sales basket analysis + objectType: + allObjectTypes: true # Should all object types be monitored? + allObjectTypesExclude: # If allObjectTypes is set to true, the object types in this array are excluded from monitoring. + # someObjectTypesInclude (below) is ignored in that case. + - table + - map + someObjectTypesInclude: # What object types should be included in monitoring? + # Only applicable if allObjectTypes is set to false. + - sheet + - barchart + - linechart + - map + appObject: + allAppObjects: true # Should all app objects be monitored? + allAppObjectsExclude: # If allAppObjects is set to true, the app objects in this array are excluded from monitoring. + # someAppObjectsInclude (below) is ignored in that case. + - objectId: AaBbCc + - objectId: DdEeFf + someAppObjectsInclude: # What app objects should be included in monitoring? + # Only applicable if allAppObjects is set to false. + - objectId: YJEpPT + method: + allMethods: true # Should all methods be monitored? + allMethodsExclude: # If allMethods is set to true, the methods in this array are excluded from monitoring. + # someMethodsInclude (below) is ignored in that case. + - Global::OpenApp + - Doc::GetAppLayout + - Doc::CreateSessionObject + someMethodsInclude: # What methods should be included in monitoring? + # Only applicable if allMethods is set to false. + - GenericObject::GetLayout + - GenericObject::GetHyperCubeContinuousData sendToMQTT: enable: false # Should log events be sent as MQTT messages? baseTopic: qliksense/logevent # What topic should log events be forwarded to? diff --git a/src/globals.js b/src/globals.js index 800139c6..10977339 100755 --- a/src/globals.js +++ b/src/globals.js @@ -694,7 +694,6 @@ class Settings { this.logger.verbose('GLOBALS: Init done'); - // eslint-disable-next-line no-constructor-return return instance; } diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 82203cf9..6604c945 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -173,6 +173,69 @@ export const confifgFileSchema = { ], }, }, + appPerformanceMonitor: { + enable: 'boolean', + appNameLookup: { + enable: 'boolean', + }, + monitorFilter: { + allApps: { + enable: 'boolean', + 'appExclude?': [ + { + 'appId?': 'string', + 'appName?': 'string', + }, + ], + objectType: { + allObjectTypes: 'boolean', + 'allObjectTypesExclude?': [], + 'someObjectTypesInclude?': [], + }, + method: { + allMethods: 'boolean', + 'allMethodsExclude?': [], + 'someMethodsInclude?': [], + }, + }, + appSpecific: { + enable: 'boolean', + app: [ + { + 'include?': [ + { + 'appId?': 'string', + 'appName?': 'string', + }, + ], + objectType: { + allObjectTypes: 'boolean', + 'allObjectTypesExclude?': [], + 'someObjectTypesInclude?': [], + }, + appObject: { + allAppObjects: 'boolean', + 'allAppObjectsExclude?': [ + { + objectId: 'string', + }, + ], + 'someAppObjectsInclude?': [ + { + objectId: 'string', + }, + ], + }, + method: { + allMethods: 'boolean', + 'allMethodsExclude?': [], + 'someMethodsInclude?': [], + }, + }, + ], + }, + }, + }, sendToMQTT: { enable: 'boolean', baseTopic: 'string', diff --git a/src/lib/config-file-verify.js b/src/lib/config-file-verify.js index 04267d25..e9fea22b 100755 --- a/src/lib/config-file-verify.js +++ b/src/lib/config-file-verify.js @@ -42,6 +42,7 @@ export async function verifyConfigFile() { process.exit(1); } + // ------------------------------ // Verify values of specific config entries // If InfluxDB is enabled, check if the version is valid diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index c0cf49a8..981c8b98 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -1,6 +1,3 @@ -/* eslint-disable prefer-destructuring */ -/* eslint-disable no-unused-vars */ - import { Point } from '@influxdata/influxdb-client'; import globals from '../globals.js'; @@ -72,7 +69,6 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server const sessionAppNamesActive = []; const storeActivedDoc = function storeActivedDoc(docID) { - // eslint-disable-next-line no-unused-vars return new Promise((resolve, _reject) => { if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) { // Session app @@ -95,10 +91,8 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server }); }; - // eslint-disable-next-line no-unused-vars const promisesActive = body.apps.active_docs.map( (docID, _idx) => - // eslint-disable-next-line no-unused-vars new Promise(async (resolve, _reject) => { await storeActivedDoc(docID); @@ -117,7 +111,6 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server const sessionAppNamesLoaded = []; const storeLoadedDoc = function storeLoadedDoc(docID) { - // eslint-disable-next-line no-unused-vars return new Promise((resolve, _reject) => { if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) { // Session app @@ -140,10 +133,8 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server }); }; - // eslint-disable-next-line no-unused-vars const promisesLoaded = body.apps.loaded_docs.map( (docID, _idx) => - // eslint-disable-next-line no-unused-vars new Promise(async (resolve, _reject) => { await storeLoadedDoc(docID); @@ -162,7 +153,6 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server const sessionAppNamesInMemory = []; const storeInMemoryDoc = function storeInMemoryDoc(docID) { - // eslint-disable-next-line no-unused-vars return new Promise((resolve, _reject) => { if (docID.substring(0, sessionAppPrefix.length) === sessionAppPrefix) { // Session app @@ -189,10 +179,8 @@ export async function postHealthMetricsToInfluxdb(serverName, host, body, server }); }; - // eslint-disable-next-line no-unused-vars const promisesInMemory = body.apps.in_memory_docs.map( (docID, _idx) => - // eslint-disable-next-line no-unused-vars new Promise(async (resolve, _reject) => { await storeInMemoryDoc(docID); @@ -723,7 +711,6 @@ export async function postUserEventToInfluxdb(msg) { globals.config.get('Butler-SOS.userEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { tags[item.tag] = item.value; } @@ -836,7 +823,6 @@ export async function postUserEventToInfluxdb(msg) { globals.config.get('Butler-SOS.userEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.userEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { point.tag(item.tag, item.value); } @@ -898,7 +884,8 @@ export async function postLogEventToInfluxdb(msg) { msg.source === 'qseow-engine' || msg.source === 'qseow-proxy' || msg.source === 'qseow-scheduler' || - msg.source === 'qseow-repository' + msg.source === 'qseow-repository' || + msg.source === 'qseow-qix-perf' ) { if (msg.source === 'qseow-engine') { tags = { @@ -908,6 +895,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -939,6 +927,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -962,6 +951,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -985,6 +975,7 @@ export async function postLogEventToInfluxdb(msg) { log_row: msg.log_row, subsystem: msg.subsystem, }; + // Tags that are empty in some cases. Only add if they are non-empty if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; @@ -1000,6 +991,39 @@ export async function postLogEventToInfluxdb(msg) { context: msg.context, raw_event: JSON.stringify(msg), }; + } else if (msg.source === 'qseow-qix-perf') { + tags = { + host: msg.host, + level: msg.level, + source: msg.source, + log_row: msg.log_row, + subsystem: msg.subsystem, + method: msg.method, + object_type: msg.object_type, + proxy_session_id: msg.proxy_session_id, + session_id: msg.session_id, + }; + + // Tags that are empty in some cases. Only add if they are non-empty + if (msg?.user_full?.length > 0) tags.user_full = msg.user_full; + if (msg?.user_directory?.length > 0) tags.user_directory = msg.user_directory; + if (msg?.user_id?.length > 0) tags.user_id = msg.user_id; + if (msg?.app_id?.length > 0) tags.app_id = msg.app_id; + if (msg?.app_name?.length > 0) tags.app_name = msg.app_name; + if (msg?.object_id?.length > 0) tags.object_id = msg.object_id; + + fields = { + app_id: msg.app_id, + process_time: msg.process_time, + work_time: msg.work_time, + lock_time: msg.lock_time, + validate_time: msg.validate_time, + traverse_time: msg.traverse_time, + handle: msg.handle, + net_ram: msg.net_ram, + peak_ram: msg.peak_ram, + raw_event: JSON.stringify(msg), + }; } // Add log event categories to tags if available @@ -1017,7 +1041,6 @@ export async function postLogEventToInfluxdb(msg) { globals.config.get('Butler-SOS.logEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { tags[item.tag] = item.value; } @@ -1056,7 +1079,8 @@ export async function postLogEventToInfluxdb(msg) { msg.source === 'qseow-engine' || msg.source === 'qseow-proxy' || msg.source === 'qseow-scheduler' || - msg.source === 'qseow-repository' + msg.source === 'qseow-repository' || + msg.source === 'qseow-qix-perf' ) { // Create new write API object // Advanced write options @@ -1205,6 +1229,37 @@ export async function postLogEventToInfluxdb(msg) { point.tag('user_directory', msg.user_directory); if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id); if (msg?.result_code?.length > 0) point.tag('result_code', msg.result_code); + } else if (msg.source === 'qseow-qix-perf') { + // Create a new point with the data to be written to InfluxDB + point = new Point('log_event') + .tag('host', msg.host) + .tag('level', msg.level) + .tag('source', msg.source) + .tag('log_row', msg.log_row) + .tag('subsystem', msg.subsystem) + .tag('method', msg.method) + .tag('object_type', msg.object_type) + .tag('proxy_session_id', msg.proxy_session_id) + .tag('session_id', msg.session_id) + .stringField('app_id', msg.app_id) + .floatField('process_time', parseFloat(msg.process_time)) + .floatField('work_time', parseFloat(msg.work_time)) + .floatField('lock_time', parseFloat(msg.lock_time)) + .floatField('validate_time', parseFloat(msg.validate_time)) + .floatField('traverse_time', parseFloat(msg.traverse_time)) + .stringField('handle', msg.handle) + .intField('net_ram', parseInt(msg.net_ram)) + .intField('peak_ram', parseInt(msg.peak_ram)) + .stringField('raw_event', JSON.stringify(msg)); + + // Tags that are empty in some cases. Only add if they are non-empty + if (msg?.user_full?.length > 0) point.tag('user_full', msg.user_full); + if (msg?.user_directory?.length > 0) + point.tag('user_directory', msg.user_directory); + if (msg?.user_id?.length > 0) point.tag('user_id', msg.user_id); + if (msg?.app_id?.length > 0) point.tag('app_id', msg.app_id); + if (msg?.app_name?.length > 0) point.tag('app_name', msg.app_name); + if (msg?.object_id?.length > 0) point.tag('object_id', msg.object_id); } // Add log event categories to tags if available @@ -1222,7 +1277,6 @@ export async function postLogEventToInfluxdb(msg) { globals.config.get('Butler-SOS.logEvents.tags').length > 0 ) { const configTags = globals.config.get('Butler-SOS.logEvents.tags'); - // eslint-disable-next-line no-restricted-syntax for (const item of configTags) { point.tag(item.tag, item.value); } diff --git a/src/lib/post-to-new-relic.js b/src/lib/post-to-new-relic.js index 1cd2d703..6074e283 100755 --- a/src/lib/post-to-new-relic.js +++ b/src/lib/post-to-new-relic.js @@ -930,7 +930,7 @@ export async function postLogEventToNewRelic(msg) { globals.logger.debug(`LOG EVENT NEW RELIC: ${msg})`); try { - // Only send log events that are enabled in the confif file + // Only send log events that are enabled in the config file if (sendNRLogEventYesNo(msg.source, msg.level) === true) { // First prepare attributes relating to the actual log event, then add attributes defined in the config file // The config file attributes can for example be used to separate data from DEV/TEST/PROD environments diff --git a/src/lib/udp_handlers_log_events.js b/src/lib/udp_handlers_log_events.js index 2e3a9417..e3effd50 100644 --- a/src/lib/udp_handlers_log_events.js +++ b/src/lib/udp_handlers_log_events.js @@ -1,5 +1,3 @@ -/* eslint-disable no-unused-vars */ - // Load global variables and functions import globals from '../globals.js'; import { postLogEventToInfluxdb } from './post-to-influxdb.js'; @@ -82,13 +80,37 @@ export function udpInitLogEventServer() { // 16: App ID associated with the event. Ex: e7af59a0-c243-480d-9571-08727551a66f // 17: Execution ID associated with the event. Ex: 4831c6a5-34f6-45bb-9d40-73a6e6992670 + // >> Message parts for log messages with Qix performance information + // 0: Message type. Always /qseow-qix-perf/ + // 1: Row number. Ex: 14 + // 2: ISO8601 formatted timestamp. Example: 20211109T193744.331+0100 + // 3: Local timezone timestamp. Example: 2021-11-09 19:37:44,331 + // 4: Log level. Possible values are: WARN, ERROR, FATAL + // 5: Hostname where the log event occured + // 6: QSEoW subsystem where log event occured. Example: System.Scheduler.Scheduler.Slave.Tasks.ReloadTask + // 7: Windows username running the originating QSEoW service. Ex: COMPANYNAME\qlikservice + // 8: Proxy session ID. Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 9: User directory of the user associated with the event. Ex: LAB + // 10: User ID of the user associated with the event. Ex: goran + // 11: Engine timestamp. Example: 2021-11-09T19:37:44.331+01:00 + // 12: Session ID. Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 13: Document ID (=app ID). Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 14: Request ID. Ex: 3b3b3b3b-3b3b-3b3b-3b3b-3b3b3b3b3b3b + // 15: Method. Ex: Global::OpenApp, Doc::GetAppLayout, GenericObject::GetLayout + // 16: Process time in milliseconds. Ex: 123 + // 17: Work time in milliseconds. Ex: 123 + // 18: Lock time in milliseconds. Ex: 123 + // 19: Validate time in milliseconds. Ex: 123 + // 20: Traverse time in milliseconds. Ex: 123 + // 21: Handle. Ex: -1, 123 + // 22: Object ID. Ex: df68e14d-1ed0-47c9-bcb6-b37a900441d8, , rwPjBk + // 23: Net RAM. Ex: 123456 bytes + // 24: Peak RAM. Ex: 123456 byets + // 25: Object type. Ex: , AppPropsList, SheetList, StoryList, VariableList, linechart, barchart, map, listbox, CurrentSelection + const msg = message.toString().split(';'); globals.logger.debug(`LOG EVENT (raw): ${message.toString()}`); - globals.logger.verbose( - `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}: ${msg[8]}` - ); - // Check if the message is a log event message we recognise // If not, log a warning and return // Take into account that msg[0] may be undefined, so check for that first @@ -97,7 +119,8 @@ export function udpInitLogEventServer() { (msg[0].toLowerCase() !== '/qseow-engine/' && msg[0].toLowerCase() !== '/qseow-proxy/' && msg[0].toLowerCase() !== '/qseow-repository/' && - msg[0].toLowerCase() !== '/qseow-scheduler/') + msg[0].toLowerCase() !== '/qseow-scheduler/' && + msg[0].toLowerCase() !== '/qseow-qix-perf/') ) { // Show warning, include first 512 characters of the message const msgShort = message.toString().substring(0, 512); @@ -116,7 +139,9 @@ export function udpInitLogEventServer() { (globals.config.get('Butler-SOS.logEvents.source.repository.enable') === true && msg[0].toLowerCase() === '/qseow-repository/') || (globals.config.get('Butler-SOS.logEvents.source.scheduler.enable') === true && - msg[0].toLowerCase() === '/qseow-scheduler/') + msg[0].toLowerCase() === '/qseow-scheduler/') || + (globals.config.get('Butler-SOS.logEvents.source.qixPerf.enable') === true && + msg[0].toLowerCase() === '/qseow-qix-perf/') ) { // Clean up the first message field (=message source) // Remove leading and trailing / @@ -134,6 +159,10 @@ export function udpInitLogEventServer() { const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; if (msg[0] === 'qseow-engine') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + // log_row: numeric // ts_iso: ISO8601 date // ts_local: ISO8601 date @@ -187,6 +216,10 @@ export function udpInitLogEventServer() { msgObj.user_full = ''; } } else if (msg[0] === 'qseow-proxy') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + msgObj = { source: msg[0], log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, @@ -216,6 +249,10 @@ export function udpInitLogEventServer() { msgObj.user_full = ''; } } else if (msg[0] === 'qseow-scheduler') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + msgObj = { source: msg[0], log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, @@ -258,6 +295,10 @@ export function udpInitLogEventServer() { msgObj.user_id = msgObj.user_full.split('\\')[1]; } } else if (msg[0] === 'qseow-repository') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, Msg: ${msg[8]}` + ); + msgObj = { source: msg[0], log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, @@ -277,6 +318,422 @@ export function udpInitLogEventServer() { context: msg[15], }; + // Different log events deliver QSEoW user directory/user differently. + // Create fields that are consistent across all log events + if (msgObj.user_directory !== '' && msgObj.user_id !== '') { + // User directory and user id available in separate fields. + // Combine them into a single field + msgObj.user_full = `${msgObj.user_directory}\\${msgObj.user_id}`; + } else { + msgObj.user_full = ''; + } + } else if (msg[0] === 'qseow-qix-perf') { + globals.logger.verbose( + `LOG EVENT: ${msg[0]}:${msg[5]}:${msg[4]}, ${msg[6]}, ${msg[9]}\\${msg[10]}, ${msg[13]}, ${msg[15]}, Object type: ${msg[25]}` + ); + + // Determine if the message should be handled, based on settings in the config file + if ( + globals.config.get('Butler-SOS.logEvents.appPerformanceMonitor.enable') === + false + ) { + globals.logger.debug( + 'LOG EVENT: Qix performance monitoring is disabled in the configuration. Skipping event.' + ); + return; + } + + // Does event match filters in the config file? + // + // There are two types of filters: + // - "All-apps" filters. Here we start with all apps and then exclude some based on filters + // - App specific filters. Here we start with an empty list of apps and then include some based on filters + // + // If an event matches any of the filters, it is accepted. + + // Get the app performance monitor filter configuration from the config file, + // so we don't have to read it every time we need some part of it + const monitorFilterConfig = globals.config.get( + 'Butler-SOS.logEvents.appPerformanceMonitor.monitorFilter' + ); + + let acceptEvent = false; + let acceptEventAppSpecific = true; + + const eventAppId = msg[13]; + let eventAppName = ''; + const eventObjectId = msg[22]; + const eventObjectType = msg[25]; + const eventMethod = msg[15]; + + // Should we get app name from the app ID? + if ( + globals.config.get( + 'Butler-SOS.logEvents.appPerformanceMonitor.appNameLookup.enable' + ) === true + ) { + // Get app name from app ID + const eventApp = globals.appNames.find((app) => app.id === eventAppId); + + if (eventApp?.name) { + eventAppName = eventApp.name; + } else { + eventAppName = ''; + } + } + + // -------------------------------------------------------- + // Check if data in event matches the app specific filters in the config file + // -------------------------------------------------------- + // monitorFilterConfig.appSpecific is an array of objects, each with following properties: + // - enable: boolean + // - app: Array of objects, each with following properties: + // - include: Array of objects, each of which has one or more of the following properties: + // - appId: string + // - appName: string + // - objectType: Object with following properties: + // - allObjectTypes: boolean + // - allObjectTypesExclude: array of strings + // - someObjectTypesInclude: array of strings + // - appObject: Object with following properties: + // - allAppObjects: boolean + // - allAppObjectsExclude: array of objects, each of which has one or more of the following properties: + // - objectId: string + // - someAppObjectsInclude: array of objects, each of which has one or more of the following properties: + // - objectId: string + // - method: Objects with following properties: + // - allMethods: boolean + // - allMethodsExclude: array of strings + // - someMethodsInclude: array of strings + // + // If the include array is empty, no apps will be accepted. + // + // If allObjectTypes is true, all object types are included, unless they are in allObjectTypesExclude. + // someObjectTypesInclude is ignored in this case. + // If allObjectTypes is false, only events matching the object types in someObjectTypesInclude are accepted. + // + // If allAppObjects is true, all objects are included, unless they are in allAppObjectsExclude. + // someAppObjectsInclude is ignored in this case. + // If allAppObjects is false, only events matching the objects in someAppObjectsInclude are accepted. + // + // If allMethods is true, all methods are included, unless they are in allMethodsExclude. + // someMethodsInclude is ignored in this case. + // If allMethods is false, only events matching the methods in someMethodsInclude are accepted. + // + + // Is app specific monitoring enabled? + if (monitorFilterConfig.appSpecific.enable === false) { + globals.logger.debug( + 'LOG EVENT: App specific monitoring is disabled in the configuration. Skipping app specific filters for this event.' + ); + acceptEventAppSpecific = false; + } + + if (acceptEventAppSpecific === true) { + // Process all app specific filters + // If one or more filter matches, the event is accepted + // If no filter matches, the event is rejected + for (const appFilter of monitorFilterConfig.appSpecific.app) { + // Check if the app ID is in the list of apps to monitor + // If not, skip the event + // The include array consists of objects with one or more of the following properties: + // - appId: string + // - appName: string + // If both appId and appName are present, both must match for the app to be included + const monitoredAppConfig = appFilter?.include?.find( + (appInclude) => + (appInclude?.appId === undefined || + appInclude.appId === eventAppId) && + (appInclude?.appName === undefined || + appInclude.appName === eventAppName) + ); + + if (monitoredAppConfig === undefined) { + // Event app ID does not match any app specific INCLUDE filters in the config file + acceptEventAppSpecific = false; + } else { + // App ID matches an app in the config file + if (appFilter.objectType.allObjectTypes === true) { + // Check if data in event matches the EXCLUDE object type filters in the config file + const excludedObjectType = + appFilter.objectType?.allObjectTypesExclude?.find( + (objectTypeExclude) => + objectTypeExclude === eventObjectType + ); + if (excludedObjectType !== undefined) { + // Object type matches an EXCLUDE object type in the config file + acceptEventAppSpecific = false; + } + } else { + // Check if data in event matches the INCLUDE object type filters in the config file + const monitoredObjectType = + appFilter.objectType?.someObjectTypesInclude?.find( + (objectTypeInclude) => + objectTypeInclude === eventObjectType + ); + if (monitoredObjectType === undefined) { + // Object type does not match an INCLUDE object type in the config file + acceptEventAppSpecific = false; + } else { + // Object type matches an INCLUDE object type in the config file + globals.logger.debug( + 'LOG EVENT: Qix performance event matches object type filters in the configuration' + ); + } + } + + // Only check object ID if the event has not been rejected so far + if (acceptEventAppSpecific === true) { + if (appFilter.appObject.allAppObjects === true) { + // Check if data in event matches the EXCLUDE object ID filters in the config file + const excludedAppObject = + appFilter.appObject?.allAppObjectsExclude?.find( + (appObjectExclude) => + appObjectExclude?.objectId === eventObjectId + ); + if (excludedAppObject !== undefined) { + // Object ID matches an EXCLUDE object ID in the config file + acceptEventAppSpecific = false; + } + } else { + // Check if data in event matches the INCLUDE object ID filters in the config file + const monitoredAppObject = + appFilter.appObject?.someAppObjectsInclude?.find( + (appObjectInclude) => + appObjectInclude?.objectId === eventObjectId + ); + if (monitoredAppObject === undefined) { + // Object ID does not match an INCLUDE object ID in the config file + acceptEventAppSpecific = false; + } + } + } + + // Only check methods if the event has not been rejected so far + if (acceptEventAppSpecific === true) { + if (appFilter.method.allMethods === true) { + // Check if data in event matches the EXCLUDE method filters in the config file + const excludedMethod = + appFilter.method?.allMethodsExclude?.find( + (methodExclude) => methodExclude === eventMethod + ); + if (excludedMethod !== undefined) { + // Method matches an EXCLUDE method in the config file + acceptEventAppSpecific = false; + } + } else { + // Check if data in event matches the INCLUDE method filters in the config file + const monitoredMethod = + appFilter.method?.someMethodsInclude?.find( + (methodInclude) => methodInclude === eventMethod + ); + if (monitoredMethod === undefined) { + // Method does not match an INCLUDE method in the config file + acceptEventAppSpecific = false; + } else { + // Method matches an INCLUDE method in the config file + globals.logger.debug( + 'LOG EVENT: Qix performance event matches method filters in the configuration' + ); + } + } + } + } + } + + // Done checking if event matches app-specific filters in the config file + + // Match on app specific filters? + if (acceptEventAppSpecific === true) { + globals.logger.debug( + 'LOG EVENT: Qix performance event matches app-specific filters in the configuration' + ); + } + } else { + acceptEventAppSpecific = false; + globals.logger.debug( + 'LOG EVENT: Qix performance event does not match app-specific filters in the configuration. Skipping app specific filters for this event.' + ); + } + + // -------------------------------------------------------- + // Check if data in event matches the "all apps" filters in the config file + // Only do this if the event has not been accepted so far + // -------------------------------------------------------- + // monitorFilterConfig.allApps is an array of objects, each with following properties: + // - enable: boolean + // - appExclude: array of objects, each of which has one or more of the following properties: + // - appId: string + // - appName: string + // - objectType: Objects with following properties: + // - allObjectTypes: boolean + // - allObjectTypesExclude: array of strings + // - someObjectTypesInclude: array of strings + // - method: Objects with following properties: + // - allMethods: boolean + // - allMethodsExclude: array of strings + // - someMethodsInclude: array of strings + // + // If appExclude is empty, all apps are included. + // If appExclude has objects, only apps not in appExclude are included. + // Matching is inclusive, i.e. if an object in the appExclude array has both appId and appName, both must match for the app to be excluded. + // + // If objectType.allObjectTypes is true, all object types are included, unless they are in allObjectTypesExclude. + // someObjectTypesInclude is ignored in this case. + // If objectType.allObjectTypes is false, only object types in someObjectTypesInclude are included. + // + // If method.allMethods is true, all methods are included, unless they are in allMethodsExclude. + // someMethodsInclude is ignored in this case. + // If method.allMethods is false, only methods in someMethodsInclude are included. + // + + let acceptEventAllApps = true; + + // Check if data in event matches the all-app filters in the config file + if (monitorFilterConfig.allApps.enable === false) { + globals.logger.debug( + 'LOG EVENT: All-apps monitoring is disabled in the configuration. Skipping all-app filters for this event.' + ); + acceptEventAllApps = false; + } else if ( + acceptEventAppSpecific === false && + monitorFilterConfig.allApps.enable === true + ) { + // Check if data in event matches the EXCLUDE app filters in the config file + if (monitorFilterConfig.allApps.appExclude?.length > 0) { + // Any matching appExclude object will cause the event to be rejected + // The appExclude array consists of objects with one or more of the following properties: + // - appId: string + // - appName: string + // If both appId and appName are present, both must match for the app to be excluded + const excludedApp = monitorFilterConfig.allApps?.appExclude.find( + (appExclude) => + (appExclude?.appId === undefined || + appExclude.appId === eventAppId) && + (appExclude?.appName === undefined || + appExclude.appName === eventAppName) + ); + if (excludedApp !== undefined) { + // App ID matches an app in the config file + acceptEventAllApps = false; + } + } + + // Only check object type if the event has not been rejected so far + if (acceptEventAllApps === true) { + if (monitorFilterConfig.allApps.objectType.allObjectTypes === true) { + // Check if data in event matches the EXCLUDE object type filters in the config file + const excludedObjectType = + monitorFilterConfig.allApps.objectType?.allObjectTypesExclude?.find( + (objectTypeExclude) => objectTypeExclude === eventObjectType + ); + if (excludedObjectType !== undefined) { + // Object type matches an object type in the config file + acceptEventAllApps = false; + } + } else { + // Check if data in event matches the INCLUDE object type filters in the config file + const monitoredObjectType = + monitorFilterConfig.allApps.objectType?.someObjectTypesInclude?.find( + (objectTypeInclude) => objectTypeInclude === eventObjectType + ); + if (monitoredObjectType === undefined) { + // Object type does not match an object type in the config file + acceptEventAllApps = false; + } + } + } + + // Only check methods if the event has not been rejected so far + if (acceptEventAllApps === true) { + if (monitorFilterConfig.allApps.method.allMethods === true) { + // Check if data in event matches the EXCLUDE method filters in the config file + const excludedMethod = + monitorFilterConfig.allApps.method?.allMethodsExclude?.find( + (methodExclude) => methodExclude === eventMethod + ); + if (excludedMethod !== undefined) { + // Method matches a method in the config file + acceptEventAllApps = false; + } + } else { + // Check if data in event matches the INCLUDE method filters in the config file + const monitoredMethod = + monitorFilterConfig.allApps.method?.someMethodsInclude?.find( + (methodInclude) => methodInclude === eventMethod + ); + if (monitoredMethod === undefined) { + // Method does not match a method in the config file + acceptEventAllApps = false; + } + } + } + + // Match on all-app filters? + if (acceptEventAllApps === true) { + acceptEvent = true; // Event matches global filters + globals.logger.debug( + 'LOG EVENT: Qix performance event matches global filters in the configuration' + ); + } + } + + // Was event accepted? + if (acceptEventAppSpecific === false && acceptEventAllApps === false) { + acceptEvent === false; + globals.logger.debug( + 'LOG EVENT: Qix performance event does not match filters in the configuration. Skipping event.' + ); + return; + } else { + acceptEvent = true; + } + + // Event matches filters in the configuration. Continue. + // Build the event object + msgObj = { + source: msg[0], + log_row: Number.isInteger(parseInt(msg[1], 10)) ? parseInt(msg[1], 10) : -1, + ts_iso: msg[2], + ts_local: msg[3], + // ts_iso: isoDateRegex.test(msg[2]) ? msg[2] : '', + // ts_local: isoDateRegex.test(msg[3]) ? msg[3] : '', + level: msg[4], + host: msg[5], + subsystem: msg[6], + windows_user: msg[7], + proxy_session_id: uuidRegex.test(msg[8]) ? msg[8] : '', + user_directory: msg[9], + user_id: msg[10], + engine_ts: msg[11], + session_id: uuidRegex.test(msg[12]) ? msg[12] : '', + app_id: uuidRegex.test(msg[13]) ? msg[13] : '', + app_name: eventAppName, + request_id: msg[14], // Request ID is an integer >= 0, set to -99 otherwise + method: msg[15], + // Processtime in float milliseconds + process_time: parseFloat(msg[16]), + work_time: parseFloat(msg[17]), + lock_time: parseFloat(msg[18]), + validate_time: parseFloat(msg[19]), + traverse_time: parseFloat(msg[20]), + // Handle is either -1 or a number. Set to -99 if not a number + handle: Number.isInteger(parseInt(msg[21], 10)) + ? parseInt(msg[21], 10) + : -99, + object_id: msg[22], + // Positive integer, set to -1 if not am integer >= 0 + net_ram: + Number.isInteger(parseInt(msg[23], 10)) && parseInt(msg[23], 10) >= 0 + ? parseInt(msg[23], 10) + : -1, + peak_ram: + Number.isInteger(parseInt(msg[24], 10)) && parseInt(msg[24], 10) >= 0 + ? parseInt(msg[24], 10) + : -1, + object_type: msg[25], + }; + // Different log events deliver QSEoW user directory/user differently. // Create fields that are consistent across all log events if (msgObj.user_directory !== '' && msgObj.user_id !== '') {