Migrate obs-infraobs-integrations to package-spec v3 #4 (#8216)

* [azure_app_service] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 2.8.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/azure_app_service * [citrix_adc] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 2.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/citrix_adc * [redis]: migration with yq * [redis] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/redis * [tomcat]: migration with yq * [tomcat] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/tomcat * Add validation.yml * Fix PR number in changelog
elastic · Oct 18, 2023 · 225555f · 225555f
1 parent d63214f
commit 225555f
Show file tree

Hide file tree

Showing 29 changed files with 202 additions and 132 deletions.
diff --git a/packages/azure_app_service/changelog.yml b/packages/azure_app_service/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 0.1.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8216
 - version: "0.0.1"
   changes:
     - description: Initial release

diff --git a/packages/azure_app_service/data_stream/app_service_logs/fields/ecs.yml b/packages/azure_app_service/data_stream/app_service_logs/fields/ecs.yml
@@ -95,22 +95,6 @@
   external: ecs
 - name: source.port
   external: ecs
-- name: geo.continent_name
-  external: ecs
-- name: geo.country_iso_code
-  external: ecs
-- name: geo.country_name
-  external: ecs
-- name: geo.location
-  external: ecs
-- name: geo.city_name
-  external: ecs
-- name: geo.region_name
-  external: ecs
-- name: geo.name
-  external: ecs
-- name: geo.region_iso_code
-  external: ecs
 - name: log.level
   external: ecs
 - name: rule.id

diff --git a/packages/azure_app_service/data_stream/app_service_logs/fields/fields.yml b/packages/azure_app_service/data_stream/app_service_logs/fields/fields.yml
@@ -129,3 +129,34 @@
     - name: log
       type: keyword
       description: Details about the event depending on level
+# The following `geo` mappings are not allowed at root level in ECS, and are defined here for backward compatibility.
+# See: https://github.com/elastic/integrations/pull/8050
+- name: geo
+  type: group
+  description: 'Geo fields can carry data about a specific location related to an event. This geolocation information can be derived from techniques such as Geo IP, or be user-supplied.'
+  fields:
+    - name: continent_name
+      type: keyword
+      description: Name of the continent.
+    - name: country_iso_code
+      type: keyword
+      description: Country ISO code.
+    - name: country_name
+      type: keyword
+      description: Country name.
+    - name: location
+      type: geo_point
+      description: Longitude and latitude.
+    - name: city_name
+      type: keyword
+      description: City name.
+    - name: region_name
+      type: keyword
+      description: Region name.
+    - name: name
+      type: keyword
+      description: 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.'
+      level: extended
+    - name: region_iso_code
+      type: keyword
+      description: Region ISO code.
diff --git a/packages/azure_app_service/manifest.yml b/packages/azure_app_service/manifest.yml
@@ -1,7 +1,7 @@
-format_version: 2.8.0
+format_version: "3.0.0"
 name: azure_app_service
 title: "Azure App Service"
-version: 0.0.1
+version: "0.1.0"
 source:
   license: "Elastic-2.0"
 description: "Collect logs and metrics from Azure App Service with Elastic Agent."
@@ -10,8 +10,10 @@ categories:
   - azure
   - cloud
 conditions:
-  kibana.version: "^8.7.1"
-  elastic.subscription: "basic"
+  kibana:
+    version: "^8.7.1"
+  elastic:
+    subscription: "basic"
 vars:
   - name: eventhub
     type: text
@@ -76,3 +78,4 @@ policy_templates:
         input_group: logs
 owner:
   github: elastic/obs-infraobs-integrations
+  type: elastic
diff --git a/packages/citrix_adc/changelog.yml b/packages/citrix_adc/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.1.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8216
 - version: "1.0.1"
   changes:
     - description: Add null check and ignore_missing check to the rename processor

diff --git a/packages/citrix_adc/data_stream/interface/fields/ecs.yml b/packages/citrix_adc/data_stream/interface/fields/ecs.yml
@@ -16,7 +16,5 @@
   name: event.type
 - external: ecs
   name: host.ip
-- external: ecs
-  name: interface.id
 - external: ecs
   name: tags
diff --git a/packages/citrix_adc/data_stream/interface/fields/fields.yml b/packages/citrix_adc/data_stream/interface/fields/fields.yml
@@ -227,3 +227,11 @@
                   description: Number of bytes transmitted by an interface since the NetScaler appliance was started or the interface statistics were cleared.
                   metric_type: counter
                   unit: byte
+# The following `interface` mappings are not allowed at root level in ECS, and are defined here for backward compatibility.
+# See: https://github.com/elastic/integrations/pull/8050
+- name: interface
+  type: group
+  fields:
+    - name: id
+      type: keyword
+      description: Interface ID as reported by an observer (typically SNMP interface ID).
diff --git a/packages/citrix_adc/manifest.yml b/packages/citrix_adc/manifest.yml
@@ -1,15 +1,17 @@
-format_version: 2.0.0
+format_version: "3.0.0"
 name: citrix_adc
 title: Citrix ADC
-version: "1.0.1"
+version: "1.1.0"
 description: This Elastic integration collects metrics from Citrix ADC product.
 type: integration
 categories:
   - observability
   - network
 conditions:
-  kibana.version: ^8.7.1
-  elastic.subscription: basic
+  kibana:
+    version: ^8.7.1
+  elastic:
+    subscription: basic
 icons:
   - src: /img/citrix_adc-logo.svg
     title: Citrix ADC logo
@@ -109,3 +111,4 @@ policy_templates:
               #    -----END CERTIFICATE-----
 owner:
   github: elastic/obs-infraobs-integrations
+  type: elastic
diff --git a/packages/citrix_adc/validation.yml b/packages/citrix_adc/validation.yml
@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - SVR00002
diff --git a/packages/redis/changelog.yml b/packages/redis/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.12.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8216
 - version: "1.11.1"
   changes:
     - description: Add null check and ignore_missing check to the rename processor

diff --git a/packages/redis/data_stream/info/fields/ecs.yml b/packages/redis/data_stream/info/fields/ecs.yml
@@ -1,7 +1,5 @@
 - external: ecs
   name: os
-- external: ecs
-  name: os.full
 - external: ecs
   name: process
 - external: ecs
@@ -10,6 +8,7 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  # Reason for adding as dimension field : A connection string to the server.
   dimension: true
 - external: ecs
   name: service.type

diff --git a/packages/redis/data_stream/info/fields/fields.yml b/packages/redis/data_stream/info/fields/fields.yml
@@ -524,11 +524,13 @@
       metric_type: gauge
       description: |
         Count of slow operations
-- name: service.address
-  type: keyword
-  # Reason for adding as dimension field : A connection string to the server.
-  dimension: true
-  description: Client address
-- name: service.version
-  type: keyword
-  description: Version of the service the data was collected from
+- name: os
+  type: group
+  description: The OS fields contain information about the operating system.
+  fields:
+    - name: full
+      type: keyword
+      description: Operating system name, including the version or code name.
+      multi_fields:
+        - name: text
+          type: match_only_text
diff --git a/packages/redis/data_stream/info/manifest.yml b/packages/redis/data_stream/info/manifest.yml
@@ -13,4 +13,4 @@ streams:
     title: Redis info metrics
     description: Collect Redis info metrics
 elasticsearch:
-  index_mode: "time_series"
+  index_mode: "time_series"
diff --git a/packages/redis/data_stream/key/manifest.yml b/packages/redis/data_stream/key/manifest.yml
@@ -22,4 +22,4 @@ streams:
     title: Redis key metrics
     description: Collect Redis key metrics
 elasticsearch:
-  index_mode: "time_series"
+  index_mode: "time_series"
diff --git a/packages/redis/data_stream/keyspace/manifest.yml b/packages/redis/data_stream/keyspace/manifest.yml
@@ -13,4 +13,4 @@ streams:
     title: Redis keyspace metrics
     description: Collect Redis keyspace metrics
 elasticsearch:
-  index_mode: "time_series"
+  index_mode: "time_series"
diff --git a/packages/redis/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/redis/data_stream/log/_dev/test/pipeline/test-common-config.yml
@@ -1,5 +1,5 @@
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"
   "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$"
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"

diff --git a/packages/redis/data_stream/log/_dev/test/pipeline/test-redis-5-0-3.log-expected.json b/packages/redis/data_stream/log/_dev/test/pipeline/test-redis-5-0-3.log-expected.json
@@ -7,7 +7,7 @@
             },
             "event": {
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2023-01-19T10:58:02.691822429Z",
+                "ingested": "2023-10-16T18:52:12.465349054Z",
                 "original": "26571:M 27 Dec 2018 11:19:18.874 * Synchronization with replica 10.114.208.18:6023 succeeded"
             },
             "log": {