Update azure package to format_version 3.0.0 (#8050)

* run 'elastic-package format' on azure package * Fixes to mappings in 'azure' package to enable migration to package-spec v3. The changes here primarily consist of: 1. Removing duplicate fields 2. Clarifying type of 'array' fields. 3. Moving root-level 'geo' fields out of external ECS definitions. This one is a little weird - ECS mandates that 'geo' fields should be nested. We get validation errors if we have them as 'external' at root level, but we need to keep them for backwards compatibility. So we add their definitions into the custom mappings files. * update azure manifest for format_version 3.0.0 * update changelog PR link * add 'validation.yml' to pass latest package spec validations * fix mappings for complex array types
elastic · Oct 16, 2023 · 3e15a1d · 3e15a1d
1 parent c9e4188
commit 3e15a1d
Show file tree

Hide file tree

Showing 47 changed files with 6,861 additions and 6,644 deletions.
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.6.0"
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8050
 - version: "1.5.33"
   changes:
     - description: Handle `json.properties.clientIp` as an alias of `json.properties.clientIP` in application gateway logs

diff --git a/packages/azure/data_stream/activitylogs/fields/agent.yml b/packages/azure/data_stream/activitylogs/fields/agent.yml
@@ -33,8 +33,6 @@
   external: ecs
 - name: host.id
   external: ecs
-- name: host.ip
-  external: ecs
 - name: host.mac
   external: ecs
 - name: host.name

diff --git a/packages/azure/data_stream/activitylogs/fields/ecs.yml b/packages/azure/data_stream/activitylogs/fields/ecs.yml
@@ -64,22 +64,6 @@
   external: ecs
 - name: source.as.organization.name
   external: ecs
-- name: geo.continent_name
-  external: ecs
-- name: geo.country_iso_code
-  external: ecs
-- name: geo.country_name
-  external: ecs
-- name: geo.location
-  external: ecs
-- name: geo.city_name
-  external: ecs
-- name: geo.region_name
-  external: ecs
-- name: geo.name
-  external: ecs
-- name: geo.region_iso_code
-  external: ecs
 - name: log.level
   external: ecs
 - name: source.geo.city_name
@@ -108,7 +92,5 @@
   external: ecs
 - name: user.id
   external: ecs
-- name: user.name
-  external: ecs
 - name: tags
   external: ecs
diff --git a/packages/azure/data_stream/activitylogs/fields/fields.yml b/packages/azure/data_stream/activitylogs/fields/fields.yml
@@ -111,3 +111,39 @@
       type: flattened
       description: |
         Event properties
+# the following `geo` mappings are not allowed at root level in ECS, and are defined here for backward compatibility.
+- name: geo
+  type: group
+  description: 'Geo fields can carry data about a specific location related to an event.
+
+    This geolocation information can be derived from techniques such as Geo IP, or be user-supplied.'
+  fields:
+    - name: continent_name
+      type: keyword
+      description: Name of the continent.
+    - name: country_iso_code
+      type: keyword
+      description: Country ISO code.
+    - name: country_name
+      type: keyword
+      description: Country name.
+    - name: location
+      type: geo_point
+      description: Longitude and latitude.
+    - name: city_name
+      type: keyword
+      description: City name.
+    - name: region_name
+      type: keyword
+      description: Region name.
+    - name: region_iso_code
+      type: keyword
+      description: Region ISO code.
+    - name: name
+      type: keyword
+      description: 'User-defined description of a location, at the level of granularity they care about.
+
+        Could be the name of their data centers, the floor number, if this describes a local physical entity, city names.
+
+        Not typically used in automated geolocation.'
+      level: extended
diff --git a/packages/azure/data_stream/activitylogs/manifest.yml b/packages/azure/data_stream/activitylogs/manifest.yml
@@ -56,4 +56,4 @@ streams:
         description: Replaces single quotes with double quotes (single quotes inside double quotes are omitted) in logs to ensure proper formatting of JSON data and avoid parsing issues during processing.
         type: bool
         multi: false
-        default: false
+        default: false
diff --git a/...eam/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json b/...eam/application_gateway/_dev/test/pipeline/test-application-gateway-raw.log-expected.json
@@ -239,9 +239,9 @@
                 "vendor": "Azure"
             },
             "related": {
-              "ip": [
-                "67.43.156.7"
-              ]
+                "ip": [
+                    "67.43.156.7"
+                ]
             },
             "rule": {
                 "category": "OWASP",
@@ -362,4 +362,4 @@
             }
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/application_gateway/manifest.yml b/packages/azure/data_stream/application_gateway/manifest.yml
@@ -24,6 +24,7 @@ streams:
         show_user: false
         description: >
           The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified.
+
       - name: tags
         type: text
         title: Tags

diff --git a/packages/azure/data_stream/auditlogs/fields/agent.yml b/packages/azure/data_stream/auditlogs/fields/agent.yml
@@ -33,8 +33,6 @@
   external: ecs
 - name: host.id
   external: ecs
-- name: host.ip
-  external: ecs
 - name: host.mac
   external: ecs
 - name: host.name

diff --git a/packages/azure/data_stream/auditlogs/fields/ecs.yml b/packages/azure/data_stream/auditlogs/fields/ecs.yml
@@ -64,16 +64,6 @@
   external: ecs
 - name: source.as.organization.name
   external: ecs
-- name: geo.continent_name
-  external: ecs
-- name: geo.country_iso_code
-  external: ecs
-- name: geo.country_name
-  external: ecs
-- name: geo.location
-  external: ecs
-- name: geo.city_name
-  external: ecs
 - name: log.level
   external: ecs
 - name: source.geo.city_name
@@ -94,8 +84,6 @@
   external: ecs
 - name: source.ip
   external: ecs
-- name: client.ip
-  external: ecs
 - name: user.full_name
   external: ecs
 - name: user.domain
@@ -104,7 +92,5 @@
   external: ecs
 - name: user.id
   external: ecs
-- name: user.name
-  external: ecs
 - name: tags
   external: ecs
diff --git a/packages/azure/data_stream/auditlogs/fields/fields.yml b/packages/azure/data_stream/auditlogs/fields/fields.yml
@@ -148,10 +148,32 @@
                     ip Address
         - name: additional_details
           type: group
-          field:
+          fields:
             - name: user_agent
               type: keyword
               description: User agent name.
         - name: authentication_protocol
           type: keyword
           description: Authentication protocol type.
+# the following `geo` mappings are not allowed at root level in ECS, and are defined here for backward compatibility.
+- name: geo
+  type: group
+  description: 'Geo fields can carry data about a specific location related to an event.
+
+    This geolocation information can be derived from techniques such as Geo IP, or be user-supplied.'
+  fields:
+    - name: continent_name
+      type: keyword
+      description: Name of the continent.
+    - name: country_iso_code
+      type: keyword
+      description: Country ISO code.
+    - name: country_name
+      type: keyword
+      description: Country name.
+    - name: location
+      type: geo_point
+      description: Longitude and latitude.
+    - name: city_name
+      type: keyword
+      description: City name.
diff --git a/packages/azure/data_stream/auditlogs/manifest.yml b/packages/azure/data_stream/auditlogs/manifest.yml
@@ -41,6 +41,7 @@ streams:
         show_user: false
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
       - name: sanitize_newlines
         type: bool
         title: Sanitizes New Lines
@@ -56,4 +57,4 @@ streams:
         description: Replaces single quotes with double quotes (single quotes inside double quotes are omitted) in logs to ensure proper formatting of JSON data and avoid parsing issues during processing.
         type: bool
         multi: false
-        default: false
+        default: false
diff --git a/packages/azure/data_stream/eventhub/fields/agent.yml b/packages/azure/data_stream/eventhub/fields/agent.yml
@@ -33,8 +33,6 @@
   external: ecs
 - name: host.id
   external: ecs
-- name: host.ip
-  external: ecs
 - name: host.mac
   external: ecs
 - name: host.name

diff --git a/packages/azure/data_stream/eventhub/fields/ecs.yml b/packages/azure/data_stream/eventhub/fields/ecs.yml
@@ -62,16 +62,6 @@
   external: ecs
 - name: source.as.organization.name
   external: ecs
-- name: geo.continent_name
-  external: ecs
-- name: geo.country_iso_code
-  external: ecs
-- name: geo.country_name
-  external: ecs
-- name: geo.location
-  external: ecs
-- name: geo.city_name
-  external: ecs
 - name: log.level
   external: ecs
 - name: source.geo.city_name
@@ -102,7 +92,5 @@
   external: ecs
 - name: user.id
   external: ecs
-- name: user.name
-  external: ecs
 - name: tags
   external: ecs
diff --git a/packages/azure/data_stream/eventhub/fields/fields.yml b/packages/azure/data_stream/eventhub/fields/fields.yml
@@ -25,3 +25,25 @@
       type: long
       description: |-
         Sequence number
+# the following `geo` mappings are not allowed at root level in ECS, and are defined here for backward compatibility.
+- name: geo
+  type: group
+  description: 'Geo fields can carry data about a specific location related to an event.
+
+    This geolocation information can be derived from techniques such as Geo IP, or be user-supplied.'
+  fields:
+    - name: continent_name
+      type: keyword
+      description: Name of the continent.
+    - name: country_iso_code
+      type: keyword
+      description: Country ISO code.
+    - name: country_name
+      type: keyword
+      description: Country name.
+    - name: location
+      type: geo_point
+      description: Longitude and latitude.
+    - name: city_name
+      type: keyword
+      description: City name.
diff --git a/packages/azure/data_stream/eventhub/manifest.yml b/packages/azure/data_stream/eventhub/manifest.yml
@@ -29,6 +29,7 @@ streams:
         title: Dataset name
         description: >
           Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
+
         type: text
       - name: storage_account_container
         type: text
@@ -38,6 +39,7 @@ streams:
         show_user: false
         description: >
           The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified.
+
       - name: tags
         type: text
         title: Tags
@@ -55,6 +57,7 @@ streams:
         show_user: false
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
       - name: sanitize_newlines
         type: bool
         title: Sanitizes New Lines
@@ -70,4 +73,4 @@ streams:
         description: Replaces single quotes with double quotes (single quotes inside double quotes are omitted) in logs to ensure proper formatting of JSON data and avoid parsing issues during processing.
         type: bool
         multi: false
-        default: false
+        default: false
diff --git a/...zure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrules-raw.log-expected.json b/...zure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrules-raw.log-expected.json
@@ -424,7 +424,7 @@
             "rule": {
                 "name": "rule",
                 "ruleset": "DNAT"
-             },
+            },
             "source": {
                 "address": "192.168.0.2",
                 "ip": "192.168.0.2",
@@ -754,4 +754,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/fields/ecs.yml b/packages/azure/data_stream/firewall_logs/fields/ecs.yml
@@ -95,22 +95,6 @@
   external: ecs
 - name: source.port
   external: ecs
-- name: geo.continent_name
-  external: ecs
-- name: geo.country_iso_code
-  external: ecs
-- name: geo.country_name
-  external: ecs
-- name: geo.location
-  external: ecs
-- name: geo.city_name
-  external: ecs
-- name: geo.region_name
-  external: ecs
-- name: geo.name
-  external: ecs
-- name: geo.region_iso_code
-  external: ecs
 - name: log.level
   external: ecs
 - name: rule.id

diff --git a/packages/azure/data_stream/firewall_logs/fields/fields.yml b/packages/azure/data_stream/firewall_logs/fields/fields.yml
@@ -50,3 +50,39 @@
       type: keyword
       description: |
         Operation name
+# the following `geo` mappings are not allowed at root level in ECS, and are defined here for backward compatibility.
+- name: geo
+  type: group
+  description: 'Geo fields can carry data about a specific location related to an event.
+
+    This geolocation information can be derived from techniques such as Geo IP, or be user-supplied.'
+  fields:
+    - name: continent_name
+      type: keyword
+      description: Name of the continent.
+    - name: country_iso_code
+      type: keyword
+      description: Country ISO code.
+    - name: country_name
+      type: keyword
+      description: Country name.
+    - name: location
+      type: geo_point
+      description: Longitude and latitude.
+    - name: city_name
+      type: keyword
+      description: City name.
+    - name: region_name
+      type: keyword
+      description: Region name.
+    - name: region_iso_code
+      type: keyword
+      description: Region ISO code.
+    - name: name
+      type: keyword
+      description: 'User-defined description of a location, at the level of granularity they care about.
+
+        Could be the name of their data centers, the floor number, if this describes a local physical entity, city names.
+
+        Not typically used in automated geolocation.'
+      level: extended