[8.0](backport #29773) ibmmq: Fix timestamp parsing (#29788)

* ibmmq: Fix timestamp parsing (#29773) This fixes the timestamp parsing in ibmmq logs: - Date processor format definition was broken for ES 8.0 (extra `a` character). - The header date format in some logs was unsupported. - The Time() field, with correct TZ and higher precission was ignored. (cherry picked from commit fbc33ab) Co-authored-by: Adrian Serrano <[email protected]>
elastic · Jan 11, 2022 · b32aa6a · b32aa6a
1 parent 44833c4
commit b32aa6a
Show file tree

Hide file tree

Showing 6 changed files with 120 additions and 118 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -97,6 +97,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of IPv6 addresses in netflow flow events. {issue}19210[19210] {pull}29383[29383]
 - Fix `sophos` KV splitting and syslog header handling {issue}24237[24237] {pull}29331[29331]
 - Undo deletion of endpoint config from cloudtrail fileset in {pull}29415[29415]. {pull}29450[29450]
+- ibmmq: Fixed `@timestamp` not being populated with correct values. {pull}29773[29773]
 
 *Heartbeat*
 

diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -25,7 +25,6 @@
     "f5.bigipafm",
     "fortinet.clientendpoint",
     "haproxy.log",
-    "ibmmq.errorlog",
     "icinga.startup",
     "imperva.securesphere",
     "infoblox.nios",

diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml
@@ -30,7 +30,7 @@ processors:
     patterns:
     - 'Process\(%{DATA:process.pid}\) User\(%{WORD:user.name}\) Program\(%{DATA:process.title}\)
       Host\(%{DATA:host.hostname}\) Installation\(%{WORD:ibmmq.errorlog.installation}\)
-      VRMF\(%{DATA:service.version}\)( QMgr\(%{DATA:ibmmq.errorlog.qmgr}\))?( Time\(%{TIMESTAMP_ISO8601:@timestamp}\))?(
+      VRMF\(%{DATA:service.version}\)( QMgr\(%{DATA:ibmmq.errorlog.qmgr}\))?( Time\(%{TIMESTAMP_ISO8601:log_timestamp}\))?(
       RemoteHost\(%{DATA:destination.address}\))?( ArithInsert1\(%{DATA:ibmmq.errorlog.arithinsert1}\))?(
       ArithInsert2\(%{DATA:ibmmq.errorlog.arithinsert2}\))?( CommentInsert1\(%{DATA:ibmmq.errorlog.commentinsert1}\))?(
       CommentInsert2\(%{DATA:ibmmq.errorlog.commentinsert2}\))?( CommentInsert3\(%{DATA:ibmmq.errorlog.commentinsert3}\))?
@@ -41,8 +41,10 @@ processors:
     field: log_timestamp
     target_field: '@timestamp'
     formats:
-    - MM/dd/yyyy hh:mm:ss aa
+    - ISO8601
+    - MM/dd/yyyy hh:mm:ss a
     - dd/MM/yyyy HH:mm:ss
+    - dd.MM.yyyy HH:mm:ss
     ignore_failure: true
 - append:
     field: ibmmq.errorlog.commentinsert

diff --git a/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01.log-expected.json b/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01.log-expected.json
@@ -66,7 +66,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2018-10-11T10:46:25.000Z",
+        "@timestamp": "2018-10-11T08:46:25.924Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -99,7 +99,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-11T10:46:26.000Z",
+        "@timestamp": "2018-10-11T08:46:26.343Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -132,7 +132,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-11T10:46:26.000Z",
+        "@timestamp": "2018-10-11T08:46:26.346Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -264,7 +264,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2018-10-28T15:12:07.000Z",
+        "@timestamp": "2018-10-28T14:12:07.685Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -297,7 +297,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-28T15:12:07.000Z",
+        "@timestamp": "2018-10-28T14:12:07.789Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -330,7 +330,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-28T15:12:08.000Z",
+        "@timestamp": "2018-10-28T14:12:08.663Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -363,7 +363,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-28T15:12:08.000Z",
+        "@timestamp": "2018-10-28T14:12:08.665Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -396,7 +396,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:48:52.000Z",
+        "@timestamp": "2018-10-29T15:48:52.594Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -429,7 +429,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:48:52.000Z",
+        "@timestamp": "2018-10-29T15:48:52.663Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -462,7 +462,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:48:53.000Z",
+        "@timestamp": "2018-10-29T15:48:53.368Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -495,7 +495,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:48:53.000Z",
+        "@timestamp": "2018-10-29T15:48:53.369Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -528,7 +528,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:49:35.000Z",
+        "@timestamp": "2018-10-29T15:49:35.477Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -561,7 +561,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:49:35.000Z",
+        "@timestamp": "2018-10-29T15:49:35.553Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -594,7 +594,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:49:36.000Z",
+        "@timestamp": "2018-10-29T15:49:36.447Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -627,7 +627,7 @@
         "user.name": "MUSR_MQADMIN"
     },
     {
-        "@timestamp": "2018-10-29T16:49:36.000Z",
+        "@timestamp": "2018-10-29T15:49:36.448Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",

diff --git a/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json b/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:00.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -33,7 +33,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:00.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -66,7 +66,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:00.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -99,7 +99,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -132,7 +132,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -165,7 +165,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -198,7 +198,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.012Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -231,7 +231,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -264,7 +264,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -297,7 +297,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -330,7 +330,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -363,7 +363,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -396,7 +396,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -429,7 +429,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -462,7 +462,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:01.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -495,7 +495,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:02.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -528,7 +528,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -561,7 +561,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.013Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -594,7 +594,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.014Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -627,7 +627,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.014Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -660,7 +660,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.014Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -693,7 +693,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.014Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",
@@ -726,7 +726,7 @@
         "user.name": "felix"
     },
     {
-        "@timestamp": "2022-01-10T10:25:59.014Z",
+        "@timestamp": "2018-07-13T07:06:03.000Z",
         "event.dataset": "ibmmq.errorlog",
         "event.kind": "event",
         "event.module": "ibmmq",