[Prometheus] Add dimension fields to query and collector datastreams …

…for TSDB support (#5974) * Add dimension fields to all datastream for TSDB support Signed-off-by: Tetiana Kravchenko <[email protected]> * add ingest pipelines, fix PR number Signed-off-by: Tetiana Kravchenko <[email protected]> * fix PR number Signed-off-by: Tetiana Kravchenko <[email protected]> * add query_name as an extra label Signed-off-by: Tetiana Kravchenko <[email protected]> * remove remote_write changes - need more investirations; replace some field, that identify where integration is runnig Signed-off-by: Tetiana Kravchenko <[email protected]> * add dimension to the agent.id fields Signed-off-by: Tetiana Kravchenko <[email protected]> --------- Signed-off-by: Tetiana Kravchenko <[email protected]>
elastic · Jun 14, 2023 · 03a618e · 03a618e
1 parent 932eb9f
commit 03a618e
Show file tree

Hide file tree

Showing 12 changed files with 65 additions and 1 deletion.
diff --git a/packages/prometheus/changelog.yml b/packages/prometheus/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.0"
+  changes:
+    - description: Add dimension fields to collector and query datastreams for TSDB support
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5974
 - version: "1.4.0"
   changes:
     - description: Add processor configuration

diff --git a/packages/prometheus/data_stream/collector/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/collector/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,8 @@
+---
+description: Pipeline for processing Prometheus metrics.
+processors:
+  - fingerprint:
+      fields: ["prometheus.labels"]
+      target_field: "prometheus.labels_id"
+      ignore_failure: true
+      ignore_missing: true
diff --git a/packages/prometheus/data_stream/collector/fields/agent.yml b/packages/prometheus/data_stream/collector/fields/agent.yml
@@ -8,6 +8,7 @@
     - name: account.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,12 +17,14 @@
     - name: availability_zone
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
     - name: instance.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
     - name: provider
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
     - name: region
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
@@ -65,6 +70,7 @@
     - name: id
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Unique container id.
     - name: image.name
@@ -133,6 +139,7 @@
     - name: name
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'Name of the host.
 

diff --git a/packages/prometheus/data_stream/collector/fields/ecs.yml b/packages/prometheus/data_stream/collector/fields/ecs.yml
@@ -2,5 +2,9 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/prometheus/data_stream/collector/fields/fields.yml b/packages/prometheus/data_stream/collector/fields/fields.yml
@@ -6,6 +6,10 @@
       object_type: keyword
       description: |
         Prometheus metric labels
+    - name: labels_id
+      type: keyword
+      dimension: true
+      description: Autogenerated ID representing the fingerprint of labels object
     - name: metrics.*
       type: object
       object_type: double

diff --git a/packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,14 @@
+---
+description: Pipeline for processing Prometheus metrics.
+processors:
+  - foreach:
+      field: "prometheus.query"
+      processor:
+        set:
+          field: "prometheus.labels.query_name"
+          value: "{{_ingest._key}}"
+  - fingerprint:
+      fields: ["prometheus.labels"]
+      target_field: "prometheus.labels_id"
+      ignore_failure: true
+      ignore_missing: true
diff --git a/packages/prometheus/data_stream/query/fields/agent.yml b/packages/prometheus/data_stream/query/fields/agent.yml
@@ -8,6 +8,7 @@
     - name: account.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,12 +17,14 @@
     - name: availability_zone
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
     - name: instance.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
     - name: provider
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
     - name: region
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
@@ -65,6 +70,7 @@
     - name: id
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Unique container id.
     - name: image.name
@@ -133,6 +139,7 @@
     - name: name
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'Name of the host.
 

diff --git a/packages/prometheus/data_stream/query/fields/ecs.yml b/packages/prometheus/data_stream/query/fields/ecs.yml
@@ -2,5 +2,9 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/prometheus/data_stream/query/fields/fields.yml b/packages/prometheus/data_stream/query/fields/fields.yml
@@ -6,6 +6,10 @@
       object_type: keyword
       description: |
         Prometheus metric labels
+    - name: labels_id
+      type: keyword
+      dimension: true
+      description: Autogenerated ID representing the fingerprint of labels object
     - name: query.*
       type: object
       object_type: double

diff --git a/packages/prometheus/data_stream/remote_write/fields/ecs.yml b/packages/prometheus/data_stream/remote_write/fields/ecs.yml
@@ -4,3 +4,5 @@
   name: service.address
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
diff --git a/packages/prometheus/docs/README.md b/packages/prometheus/docs/README.md
@@ -182,6 +182,7 @@ The fields reported are:
 | Field | Description | Type |
 |---|---|---|
 | @timestamp | Event timestamp. | date |
+| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
 | cloud.availability_zone | Availability zone in which this host is running. | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |
@@ -223,6 +224,7 @@ The fields reported are:
 | prometheus.\*.rate | Prometheus rated counter metric | object |
 | prometheus.\*.value | Prometheus gauge metric | object |
 | prometheus.labels.\* | Prometheus metric labels | object |
+| prometheus.labels_id | Autogenerated ID representing the fingerprint of labels object | keyword |
 | prometheus.metrics.\* | Prometheus metric | object |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |
 | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
@@ -394,6 +396,7 @@ The fields reported are:
 | Field | Description | Type |
 |---|---|---|
 | @timestamp | Event timestamp. | date |
+| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
 | cloud.availability_zone | Availability zone in which this host is running. | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |
@@ -615,6 +618,7 @@ The fields reported are:
 | Field | Description | Type |
 |---|---|---|
 | @timestamp | Event timestamp. | date |
+| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
 | cloud.availability_zone | Availability zone in which this host is running. | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |
@@ -652,6 +656,7 @@ The fields reported are:
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
 | prometheus.labels.\* | Prometheus metric labels | object |
+| prometheus.labels_id | Autogenerated ID representing the fingerprint of labels object | keyword |
 | prometheus.query.\* | Prometheus value resulted from PromQL | object |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |
 | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |

diff --git a/packages/prometheus/manifest.yml b/packages/prometheus/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: prometheus
 title: Prometheus
-version: 1.4.0
+version: 1.5.0
 license: basic
 description: Collect metrics from Prometheus servers with Elastic Agent.
 type: integration