[GCP] Add dimensions for metrics data streams (#8314)

* add dimensions mappings changes gcp field type to group add labels ffingerprint in ingest pipeline * update docs * bump package version * update dns docs * fix compute pipeline test * change 60s to 5m storage period * remove metrics fingerprint * update docs * add agent id ecs field * update docs * remove empty dimensions * update cloudrun docs * remove redundant dimensions * revert firestore
elastic · Dec 6, 2023 · 98f4a01 · 98f4a01
1 parent acb9d7e
commit 98f4a01
Show file tree

Hide file tree

Showing 49 changed files with 216 additions and 4 deletions.
diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.32.1"
+  changes:
+    - description: Add dimensions mappings and the metrics_fingerprint field across all metrics data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8314
 - version: "2.32.0"
   changes:
     - description: Add new billing data stream fields.

diff --git a/packages/gcp/data_stream/cloudrun_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudrun_metrics/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP Cloud Run metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - rename:
       field: gcp.metrics.container.billable_instance_time.value
       target_field: gcp.cloudrun_metrics.container.billable_instance_time

diff --git a/packages/gcp/data_stream/cloudrun_metrics/fields/ecs.yml b/packages/gcp/data_stream/cloudrun_metrics/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/cloudrun_metrics/fields/package-fields.yml b/packages/gcp/data_stream/cloudrun_metrics/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP CloudSQL MySQL metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - drop:
       description: Drop if database is not MySQL.
       if: "ctx?.gcp?.labels?.cloudsql?.name != 'mysql'"

diff --git a/packages/gcp/data_stream/cloudsql_mysql/fields/ecs.yml b/packages/gcp/data_stream/cloudsql_mysql/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/cloudsql_mysql/fields/package-fields.yml b/packages/gcp/data_stream/cloudsql_mysql/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP CloudSQL PostgreSQL metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - drop:
       description: Drop if database is not PostgreSQL.
       if: "ctx?.gcp?.labels?.cloudsql?.name != 'postgres'"

diff --git a/packages/gcp/data_stream/cloudsql_postgresql/fields/ecs.yml b/packages/gcp/data_stream/cloudsql_postgresql/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/cloudsql_postgresql/fields/package-fields.yml b/packages/gcp/data_stream/cloudsql_postgresql/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/cloudsql_sqlserver/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_sqlserver/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP CloudSQL SQL Server metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - drop:
       description: Drop if database is not SQLServer.
       if: "ctx?.gcp?.labels?.cloudsql?.name != 'sqlserver'"

diff --git a/packages/gcp/data_stream/cloudsql_sqlserver/fields/ecs.yml b/packages/gcp/data_stream/cloudsql_sqlserver/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/cloudsql_sqlserver/fields/package-fields.yml b/packages/gcp/data_stream/cloudsql_sqlserver/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/compute/_dev/test/pipeline/test-gcp-compute.json-expected.json b/packages/gcp/data_stream/compute/_dev/test/pipeline/test-gcp-compute.json-expected.json
@@ -70,7 +70,8 @@
                     "user": {
                         "goog-gke-node": ""
                     }
-                }
+                },
+                "labels_fingerprint": "vP7l3YwKK9Mr+96HYpEj/Of6zVI="
             },
             "host": {
                 "id": "4751091017865185079",

diff --git a/packages/gcp/data_stream/compute/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/compute/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP Compute metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - rename:
       field: gcp.metrics.firewall.dropped.bytes
       target_field: gcp.compute.firewall.dropped.bytes

diff --git a/packages/gcp/data_stream/compute/fields/ecs.yml b/packages/gcp/data_stream/compute/fields/ecs.yml
@@ -2,18 +2,23 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
   name: cloud.availability_zone
+  dimension: true
 - external: ecs
   name: cloud.instance.id
+  dimension: true
 - external: ecs
   name: cloud.machine.type
+  dimension: true
 - external: ecs
   name: cloud.provider
 - external: ecs
   name: cloud.region
+  dimension: true
 - external: ecs
   name: ecs.version
 - external: ecs
@@ -22,3 +27,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/compute/fields/package-fields.yml b/packages/gcp/data_stream/compute/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP Dataproc metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - rename:
       field: gcp.metrics.batch.spark.executors.count
       target_field: gcp.dataproc.batch.spark.executors.count

diff --git a/packages/gcp/data_stream/dataproc/fields/ecs.yml b/packages/gcp/data_stream/dataproc/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/dataproc/fields/package-fields.yml b/packages/gcp/data_stream/dataproc/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/firestore/fields/ecs.yml b/packages/gcp/data_stream/firestore/fields/ecs.yml
@@ -21,4 +21,4 @@
 - external: ecs
   name: error.message
 - external: ecs
-  name: service.type
+  name: service.type
diff --git a/packages/gcp/data_stream/gke/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/gke/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP GKE metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - rename:
       field: gcp.metrics.container.cpu.core_usage_time.sec
       target_field: gcp.gke.container.cpu.core_usage_time.sec

diff --git a/packages/gcp/data_stream/gke/fields/ecs.yml b/packages/gcp/data_stream/gke/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/gke/fields/package-fields.yml b/packages/gcp/data_stream/gke/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/loadbalancing_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/loadbalancing_metrics/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP Loadbalancing metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - rename:
       field: gcp.metrics.https.backend_request.bytes
       target_field: gcp.loadbalancing_metrics.https.backend_request.bytes

diff --git a/packages/gcp/data_stream/loadbalancing_metrics/fields/ecs.yml b/packages/gcp/data_stream/loadbalancing_metrics/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/gcp/data_stream/loadbalancing_metrics/fields/package-fields.yml b/packages/gcp/data_stream/loadbalancing_metrics/fields/package-fields.yml
@@ -1,7 +1,13 @@
 - name: gcp
+  type: group
   description: >-
     GCP module
   fields:
+    - name: labels_fingerprint
+      dimension: true
+      type: keyword
+      description: >-
+        Hashed value of the labels field.
     - name: labels
       type: group
       description: >-

diff --git a/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml
@@ -1,6 +1,11 @@
 ---
 description: Pipeline for parsing GCP PubSub metrics.
 processors:
+  - fingerprint:
+      fields:
+        - gcp.labels
+      target_field: gcp.labels_fingerprint
+      ignore_missing: true
   - rename:
       field: gcp.metrics.snapshot.backlog.bytes
       target_field: gcp.pubsub.snapshot.backlog.bytes

diff --git a/packages/gcp/data_stream/pubsub/fields/ecs.yml b/packages/gcp/data_stream/pubsub/fields/ecs.yml
@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -22,3 +23,6 @@
   name: error.message
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true