From 9883702f3c57e72848598f66a1579e903a6b0896 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 12 Nov 2024 15:50:15 +0100 Subject: [PATCH] [docker] use ecs definition of the 'event.dataset' field for container_logs (#11672) * Use ecs definition of the 'event.dataset' field for container_logs Signed-off-by: Tetiana Kravchenko * change pr link; fix field description Signed-off-by: Tetiana Kravchenko * Use ecs definition of the 'event.dataset' field for all datastreams Signed-off-by: Tetiana Kravchenko * remove empty line Signed-off-by: Tetiana Kravchenko --------- Signed-off-by: Tetiana Kravchenko --- packages/docker/changelog.yml | 5 +++++ .../container/fields/base-fields.yml | 4 ---- .../data_stream/container/fields/ecs.yml | 2 ++ .../container_logs/fields/base-fields.yml | 4 ---- .../data_stream/container_logs/fields/ecs.yml | 2 ++ .../data_stream/cpu/fields/base-fields.yml | 4 ---- .../docker/data_stream/cpu/fields/ecs.yml | 2 ++ .../data_stream/diskio/fields/base-fields.yml | 4 ---- .../docker/data_stream/diskio/fields/ecs.yml | 2 ++ .../data_stream/event/fields/base-fields.yml | 4 ---- .../docker/data_stream/event/fields/ecs.yml | 2 ++ .../healthcheck/fields/base-fields.yml | 4 ---- .../data_stream/healthcheck/fields/ecs.yml | 2 ++ .../data_stream/image/fields/base-fields.yml | 4 ---- .../docker/data_stream/image/fields/ecs.yml | 2 ++ .../data_stream/info/fields/base-fields.yml | 4 ---- .../docker/data_stream/info/fields/ecs.yml | 2 ++ .../data_stream/memory/fields/base-fields.yml | 4 ---- .../docker/data_stream/memory/fields/ecs.yml | 2 ++ .../network/fields/base-fields.yml | 4 ---- .../docker/data_stream/network/fields/ecs.yml | 2 ++ packages/docker/docs/README.md | 20 +++++++++---------- packages/docker/manifest.yml | 2 +- 23 files changed, 36 insertions(+), 51 deletions(-) diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index 03c88f04f6c..2739868a2e7 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: 2.12.0 + changes: + - description: Use ecs definition of the 'event.dataset' field. + type: enhancement + link: https://github.com/elastic/integrations/pull/11672 - version: 2.11.0 changes: - description: Bump package-spec version to 3.2.2 to run on Serverless and stack version 9.0. diff --git a/packages/docker/data_stream/container/fields/base-fields.yml b/packages/docker/data_stream/container/fields/base-fields.yml index a6058da36cd..d1bacfeef9e 100644 --- a/packages/docker/data_stream/container/fields/base-fields.yml +++ b/packages/docker/data_stream/container/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.container diff --git a/packages/docker/data_stream/container/fields/ecs.yml b/packages/docker/data_stream/container/fields/ecs.yml index 06836576c29..e2dafbfba63 100644 --- a/packages/docker/data_stream/container/fields/ecs.yml +++ b/packages/docker/data_stream/container/fields/ecs.yml @@ -57,3 +57,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/container_logs/fields/base-fields.yml b/packages/docker/data_stream/container_logs/fields/base-fields.yml index 16bef6cc9a4..a10370f9b70 100644 --- a/packages/docker/data_stream/container_logs/fields/base-fields.yml +++ b/packages/docker/data_stream/container_logs/fields/base-fields.yml @@ -14,10 +14,6 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.container_logs - name: log.offset type: long description: Offset of the entry in the log file. diff --git a/packages/docker/data_stream/container_logs/fields/ecs.yml b/packages/docker/data_stream/container_logs/fields/ecs.yml index 471f0cb8582..0901aaf13a0 100644 --- a/packages/docker/data_stream/container_logs/fields/ecs.yml +++ b/packages/docker/data_stream/container_logs/fields/ecs.yml @@ -36,3 +36,5 @@ name: host.os.version - external: ecs name: host.type +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/cpu/fields/base-fields.yml b/packages/docker/data_stream/cpu/fields/base-fields.yml index 93d8b9abeb3..d1bacfeef9e 100644 --- a/packages/docker/data_stream/cpu/fields/base-fields.yml +++ b/packages/docker/data_stream/cpu/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.cpu diff --git a/packages/docker/data_stream/cpu/fields/ecs.yml b/packages/docker/data_stream/cpu/fields/ecs.yml index b9ee5bd70c4..49459dd998f 100644 --- a/packages/docker/data_stream/cpu/fields/ecs.yml +++ b/packages/docker/data_stream/cpu/fields/ecs.yml @@ -64,3 +64,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/diskio/fields/base-fields.yml b/packages/docker/data_stream/diskio/fields/base-fields.yml index 05f6d800b3e..d1bacfeef9e 100644 --- a/packages/docker/data_stream/diskio/fields/base-fields.yml +++ b/packages/docker/data_stream/diskio/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.diskio diff --git a/packages/docker/data_stream/diskio/fields/ecs.yml b/packages/docker/data_stream/diskio/fields/ecs.yml index b2f3a3e160b..6ee8b873d94 100644 --- a/packages/docker/data_stream/diskio/fields/ecs.yml +++ b/packages/docker/data_stream/diskio/fields/ecs.yml @@ -70,3 +70,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/event/fields/base-fields.yml b/packages/docker/data_stream/event/fields/base-fields.yml index 8876f69d414..d1bacfeef9e 100644 --- a/packages/docker/data_stream/event/fields/base-fields.yml +++ b/packages/docker/data_stream/event/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.event diff --git a/packages/docker/data_stream/event/fields/ecs.yml b/packages/docker/data_stream/event/fields/ecs.yml index 471f0cb8582..0901aaf13a0 100644 --- a/packages/docker/data_stream/event/fields/ecs.yml +++ b/packages/docker/data_stream/event/fields/ecs.yml @@ -36,3 +36,5 @@ name: host.os.version - external: ecs name: host.type +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/healthcheck/fields/base-fields.yml b/packages/docker/data_stream/healthcheck/fields/base-fields.yml index 29740d87515..d1bacfeef9e 100644 --- a/packages/docker/data_stream/healthcheck/fields/base-fields.yml +++ b/packages/docker/data_stream/healthcheck/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.healthcheck diff --git a/packages/docker/data_stream/healthcheck/fields/ecs.yml b/packages/docker/data_stream/healthcheck/fields/ecs.yml index 06836576c29..e2dafbfba63 100644 --- a/packages/docker/data_stream/healthcheck/fields/ecs.yml +++ b/packages/docker/data_stream/healthcheck/fields/ecs.yml @@ -57,3 +57,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/image/fields/base-fields.yml b/packages/docker/data_stream/image/fields/base-fields.yml index 55af58edf25..d1bacfeef9e 100644 --- a/packages/docker/data_stream/image/fields/base-fields.yml +++ b/packages/docker/data_stream/image/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.image diff --git a/packages/docker/data_stream/image/fields/ecs.yml b/packages/docker/data_stream/image/fields/ecs.yml index c8a45728f8b..3e79a80fe38 100644 --- a/packages/docker/data_stream/image/fields/ecs.yml +++ b/packages/docker/data_stream/image/fields/ecs.yml @@ -56,3 +56,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/info/fields/base-fields.yml b/packages/docker/data_stream/info/fields/base-fields.yml index 37248867c94..d1bacfeef9e 100644 --- a/packages/docker/data_stream/info/fields/base-fields.yml +++ b/packages/docker/data_stream/info/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.info diff --git a/packages/docker/data_stream/info/fields/ecs.yml b/packages/docker/data_stream/info/fields/ecs.yml index c8a45728f8b..3e79a80fe38 100644 --- a/packages/docker/data_stream/info/fields/ecs.yml +++ b/packages/docker/data_stream/info/fields/ecs.yml @@ -56,3 +56,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/memory/fields/base-fields.yml b/packages/docker/data_stream/memory/fields/base-fields.yml index 2ce29ef6347..d1bacfeef9e 100644 --- a/packages/docker/data_stream/memory/fields/base-fields.yml +++ b/packages/docker/data_stream/memory/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.memory diff --git a/packages/docker/data_stream/memory/fields/ecs.yml b/packages/docker/data_stream/memory/fields/ecs.yml index 56f04dcbe85..155ffdbeb59 100644 --- a/packages/docker/data_stream/memory/fields/ecs.yml +++ b/packages/docker/data_stream/memory/fields/ecs.yml @@ -64,3 +64,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/data_stream/network/fields/base-fields.yml b/packages/docker/data_stream/network/fields/base-fields.yml index 20f04fed255..d1bacfeef9e 100644 --- a/packages/docker/data_stream/network/fields/base-fields.yml +++ b/packages/docker/data_stream/network/fields/base-fields.yml @@ -14,7 +14,3 @@ type: constant_keyword description: Event module value: docker -- name: event.dataset - type: constant_keyword - description: Event dataset - value: docker.network diff --git a/packages/docker/data_stream/network/fields/ecs.yml b/packages/docker/data_stream/network/fields/ecs.yml index 57e5a4f0151..0ac561b7cf3 100644 --- a/packages/docker/data_stream/network/fields/ecs.yml +++ b/packages/docker/data_stream/network/fields/ecs.yml @@ -69,3 +69,5 @@ - external: ecs name: cloud.instance.id dimension: true +- external: ecs + name: event.dataset diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index 595135ed439..b17bd9a0bd0 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -86,7 +86,7 @@ running Docker containers. | docker.container.status | Container status. | keyword | | | docker.container.tags | Image tags. | keyword | | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.dataset | Event dataset | constant_keyword | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | event.module | Event module | constant_keyword | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | host.architecture | Operating system architecture. | keyword | | @@ -209,7 +209,7 @@ The Docker `cpu` data stream collects runtime CPU metrics. | docker.cpu.user.pct | Percentage of time in user space. | scaled_float | percent | gauge | | docker.cpu.user.ticks | CPU ticks in user space. | long | | counter | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| event.dataset | Event dataset | constant_keyword | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | | event.module | Event module | constant_keyword | | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | | host.architecture | Operating system architecture. | keyword | | | @@ -400,7 +400,7 @@ The Docker `diskio` data stream collects disk I/O metrics. | docker.diskio.write.service_time | Total time to service IO requests, in nanoseconds | long | | counter | | docker.diskio.write.wait_time | Total time requests spent waiting in queues for service, in nanoseconds | long | | counter | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| event.dataset | Event dataset | constant_keyword | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | | event.module | Event module | constant_keyword | | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | | host.architecture | Operating system architecture. | keyword | | | @@ -502,7 +502,7 @@ The Docker `event` data stream collects docker events | docker.event.status | Event status | keyword | | docker.event.type | The type of object emitting the event | keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Event dataset | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | event.module | Event module | constant_keyword | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | host.architecture | Operating system architecture. | keyword | @@ -590,7 +590,7 @@ docker `HEALTHCHECK` instruction has been used to build the docker image. | docker.healthcheck.failingstreak | concurent failed check | integer | counter | | docker.healthcheck.status | Healthcheck status code | keyword | | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.dataset | Event dataset | constant_keyword | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | event.module | Event module | constant_keyword | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | host.architecture | Operating system architecture. | keyword | | @@ -704,7 +704,7 @@ The Docker `image` data stream collects metrics on docker images | docker.image.size.virtual | Size of the image. | long | gauge | | docker.image.tags | Image tags. | keyword | | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.dataset | Event dataset | constant_keyword | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | event.module | Event module | constant_keyword | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | host.architecture | Operating system architecture. | keyword | | @@ -801,7 +801,7 @@ https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/display-s | docker.info.id | Unique Docker host identifier. | keyword | | | docker.info.images | Total number of existing images. | long | counter | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.dataset | Event dataset | constant_keyword | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | event.module | Event module | constant_keyword | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | host.architecture | Operating system architecture. | keyword | | @@ -890,7 +890,7 @@ The Docker `memory` data stream collects memory metrics from docker. | docker.memory.usage.pct | Memory usage percentage. | scaled_float | percent | gauge | | docker.memory.usage.total | Total memory usage. | long | byte | gauge | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| event.dataset | Event dataset | constant_keyword | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | | event.module | Event module | constant_keyword | | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | | host.architecture | Operating system architecture. | keyword | | | @@ -1025,7 +1025,7 @@ The Docker `network` data stream collects network metrics. | docker.network.outbound.errors | Total errors on outgoing packets. | long | counter | | docker.network.outbound.packets | Total number of outgoing packets. | long | counter | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.dataset | Event dataset | constant_keyword | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | event.module | Event module | constant_keyword | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | | host.architecture | Operating system architecture. | keyword | | @@ -1132,7 +1132,7 @@ The Docker `container_logs` data stream collects container logs. | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Event dataset | constant_keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | event.module | Event module | constant_keyword | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | host.architecture | Operating system architecture. | keyword | diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index ef3189161d7..1ebb12851bf 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.11.0 +version: 2.12.0 description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: