[PSQL] updating postgresql ECS version and adding event.original opti…

…ons (#1095) * updating postgresql ECS version and adding event.original options * update manifest/changelog and linting * regenerate * linting * update version bump
elastic · Jun 9, 2021 · 2e0040a · 2e0040a
1 parent 427d99e
commit 2e0040a
Show file tree

Hide file tree

Showing 49 changed files with 2,852 additions and 1,149 deletions.
diff --git a/packages/postgresql/changelog.yml b/packages/postgresql/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.0"
+  changes:
+    - description: update to ECS 1.10.0 and adding event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1095
 - version: "0.3.1"
   changes:
     - description: update to ECS 1.9.0

diff --git a/...test-postgresql-10-default.log-config.yml → ..._dev/test/pipeline/test-common-config.yml b/...test-postgresql-10-default.log-config.yml → ..._dev/test/pipeline/test-common-config.yml
@@ -2,3 +2,6 @@ dynamic_fields:
   event.ingested: ".*"
 multiline:
   first_line_pattern: '^\d{4}-\d{2}-\d{2} '
+fields:
+  tags:
+    - preserve_original_event
diff --git a/...ostgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-expected.json b/...ostgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-expected.json
@@ -5,6 +5,9 @@
                 "pid": 23922
             },
             "@timestamp": "2020-04-15T10:02:55.244Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:02:55.244 CEST"
@@ -15,7 +18,8 @@
             },
             "message": "database system was shut down at 2020-04-15 12:02:52 CEST",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997366991Z",
+                "ingested": "2021-06-09T13:07:40.893758900Z",
+                "original": "2020-04-15 12:02:55.244 CEST [23922] LOG:  database system was shut down at 2020-04-15 12:02:52 CEST",
                 "category": [
                     "database"
                 ],
@@ -24,13 +28,19 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "process": {
                 "pid": 23920
             },
             "@timestamp": "2020-04-15T10:02:55.247Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:02:55.247 CEST"
@@ -41,7 +51,8 @@
             },
             "message": "database system is ready to accept connections",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997369966Z",
+                "ingested": "2021-06-09T13:07:40.893782500Z",
+                "original": "2020-04-15 12:02:55.247 CEST [23920] LOG:  database system is ready to accept connections",
                 "category": [
                     "database"
                 ],
@@ -50,13 +61,19 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "process": {
                 "pid": 24981
             },
             "@timestamp": "2020-04-15T10:04:45.416Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:04:45.416 CEST"
@@ -67,7 +84,8 @@
             },
             "message": "password authentication failed for user \"root\"",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997370905Z",
+                "ingested": "2021-06-09T13:07:40.893789900Z",
+                "original": "2020-04-15 12:04:45.416 CEST [24981] FATAL:  password authentication failed for user \"root\"",
                 "category": [
                     "database"
                 ],
@@ -76,13 +94,19 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "process": {
                 "pid": 24981
             },
             "@timestamp": "2020-04-15T10:04:45.416Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:04:45.416 CEST"
@@ -93,7 +117,8 @@
             },
             "message": "Role \"root\" does not exist.\n\tConnection matched pg_hba.conf line 80: \"local   all             all                                     md5\"",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997377080Z",
+                "ingested": "2021-06-09T13:07:40.893797900Z",
+                "original": "2020-04-15 12:04:45.416 CEST [24981] DETAIL:  Role \"root\" does not exist.\n\tConnection matched pg_hba.conf line 80: \"local   all             all                                     md5\"",
                 "category": [
                     "database"
                 ],
@@ -102,13 +127,19 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "process": {
                 "pid": 24981
             },
             "@timestamp": "2020-04-15T10:04:45.416Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:04:45.416 CEST"
@@ -119,7 +150,8 @@
             },
             "message": "could not send data to client: Broken pipe",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997437213Z",
+                "ingested": "2021-06-09T13:07:40.893826700Z",
+                "original": "2020-04-15 12:04:45.416 CEST [24981] LOG:  could not send data to client: Broken pipe",
                 "category": [
                     "database"
                 ],
@@ -128,13 +160,19 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "process": {
                 "pid": 25143
             },
             "@timestamp": "2020-04-15T10:06:36.719Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:06:36.719 CEST"
@@ -145,7 +183,8 @@
             },
             "message": "syntax error at or near \"l\" at character 1",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997440778Z",
+                "ingested": "2021-06-09T13:07:40.893836800Z",
+                "original": "2020-04-15 12:06:36.719 CEST [25143] ERROR:  syntax error at or near \"l\" at character 1",
                 "category": [
                     "database"
                 ],
@@ -154,13 +193,19 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "process": {
                 "pid": 25143
             },
             "@timestamp": "2020-04-15T10:56:29.569Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "postgresql": {
                 "log": {
                     "timestamp": "2020-04-15 12:56:29.569 CEST"
@@ -171,7 +216,8 @@
             },
             "message": "SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,\n\t        aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,\n\t        al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName\n\t         FROM public.rc_audit_log_events AS al\n\t        LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id\n\t            LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id\n\t            LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id\n\t            LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id\n\t            LEFT JOIN rc_subjects AS s ON s.id=al.subject_id\n\t            LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id\n\t            LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id\n\t            WHERE al.tenant_id=$1 AND al.study_id=$2  AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC  limit $8\n\t;",
             "event": {
-                "ingested": "2021-04-23T12:57:22.997441657Z",
+                "ingested": "2021-06-09T13:07:40.893843700Z",
+                "original": "2020-04-15 12:56:29.569 CEST [25143] STATEMENT:  SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,\n\t        aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,\n\t        al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName\n\t         FROM public.rc_audit_log_events AS al\n\t        LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id\n\t            LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id\n\t            LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id\n\t            LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id\n\t            LEFT JOIN rc_subjects AS s ON s.id=al.subject_id\n\t            LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id\n\t            LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id\n\t            WHERE al.tenant_id=$1 AND al.study_id=$2  AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC  limit $8\n\t;",
                 "category": [
                     "database"
                 ],
@@ -180,7 +226,10 @@
                 ],
                 "timezone": "CEST",
                 "kind": "event"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/...ta_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-config.yml b/...ta_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-config.yml