[Azure Logs] Fix event.original handling on Application Gateway and…

… the generic Event Hub integration (elastic#5361)

* Fix `event.original` handling

Some event forwarders (for example, Logstash) send both `message` and
`event.original` fields, causing an error in the ingest pipeline.

With this change, the pipeline ensures there will only be the
`event.original` field early in the pipeline.

packages/azure/changelog.yml

@@ -1,8 +1,13 @@
+- version: "1.5.10"
+  changes:
+    - description: Check for 'event.original' already existing in Application Gateway and Event Hub ingest pipelines
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5361
 - version: "1.5.9"
   changes:
-      - description: Check for 'event.original' already existing in firewall logs ingest pipeline
-        type: bugfix
-        link: https://github.com/elastic/integrations/pull/5334
+    - description: Check for 'event.original' already existing in firewall logs ingest pipeline
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5334
 - version: "1.5.8"
   changes:
     - description: Add `storage_account_container` option to the Application Gateway integration

packages/azure/data_stream/application_gateway/elasticsearch/ingest_pipeline/default.yml

@@ -17,6 +17,14 @@ processors:
       field: message
       target_field: event.original
       ignore_missing: true
+      if: 'ctx.event?.original == null'
+      description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.'
+  - remove:
+      field: message
+      ignore_missing: true
+      if: 'ctx.event?.original != null'
+      description: 'The `message` field is no longer required if the document has an `event.original` field.'
+
   - json:
       field: event.original
       target_field: json

packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/parsed-message.yml

@@ -5,6 +5,13 @@ processors:
       field: message
       target_field: event.original
       ignore_missing: true
+      if: 'ctx.event?.original == null'
+      description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.'
+  - remove:
+      field: message
+      ignore_missing: true
+      if: 'ctx.event?.original != null'
+      description: 'The `message` field is no longer required if the document has an `event.original` field.'
   - json:
       field: event.original
       target_field: azure.eventhub

packages/azure/manifest.yml

@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: 1.5.9
+version: 1.5.10
 release: ga
 description: This Elastic integration collects logs from Azure
 type: integration