From 9cf1dff90a0b986eb7353e5631687cb993759631 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 24 Nov 2022 12:38:54 +0200 Subject: [PATCH 1/6] Add docker_logs datastream for docker container logs collection Signed-off-by: ChrsMark --- packages/docker/changelog.yml | 5 + .../docker_logs/agent/stream/stream.yml.hbs | 18 ++++ .../docker_logs/fields/base-fields.yml | 20 ++++ .../data_stream/docker_logs/fields/ecs.yml | 38 +++++++ .../data_stream/docker_logs/fields/fields.yml | 42 ++++++++ .../data_stream/docker_logs/manifest.yml | 56 ++++++++++ .../data_stream/docker_logs/sample_event.json | 102 ++++++++++++++++++ packages/docker/manifest.yml | 9 +- 8 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 packages/docker/data_stream/docker_logs/agent/stream/stream.yml.hbs create mode 100644 packages/docker/data_stream/docker_logs/fields/base-fields.yml create mode 100644 packages/docker/data_stream/docker_logs/fields/ecs.yml create mode 100644 packages/docker/data_stream/docker_logs/fields/fields.yml create mode 100644 packages/docker/data_stream/docker_logs/manifest.yml create mode 100644 packages/docker/data_stream/docker_logs/sample_event.json diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index e3b8a3b5a88..443436f751d 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.0" + changes: + - description: Add docker_logs + type: enhancement + link: http://github.com/elastic/integrations/pull/9999 - version: "2.2.0" changes: - description: Add skip_major flag diff --git a/packages/docker/data_stream/docker_logs/agent/stream/stream.yml.hbs b/packages/docker/data_stream/docker_logs/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..63fb1152bf7 --- /dev/null +++ b/packages/docker/data_stream/docker_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,18 @@ +id: docker-container-logs-${docker.container.name}-${docker.container.id} +paths: +{{#each paths}} + - {{this}} +{{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} +parsers: +- container: + stream: {{ containerParserStream }} + format: docker +{{ additionalParsersConfig }} + +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/docker/data_stream/docker_logs/fields/base-fields.yml b/packages/docker/data_stream/docker_logs/fields/base-fields.yml new file mode 100644 index 00000000000..c6fcca2192b --- /dev/null +++ b/packages/docker/data_stream/docker_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: docker +- name: event.dataset + type: constant_keyword + description: Event dataset + value: docker.docker_logs diff --git a/packages/docker/data_stream/docker_logs/fields/ecs.yml b/packages/docker/data_stream/docker_logs/fields/ecs.yml new file mode 100644 index 00000000000..471f0cb8582 --- /dev/null +++ b/packages/docker/data_stream/docker_logs/fields/ecs.yml @@ -0,0 +1,38 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: container.id +- external: ecs + name: container.name +- external: ecs + name: container.runtime +- external: ecs + name: container.image.name +- external: ecs + name: host +- external: ecs + name: host.architecture +- external: ecs + name: host.ip +- external: ecs + name: host.mac +- external: ecs + name: host.name +- external: ecs + name: host.os.family +- external: ecs + name: host.os.full +- external: ecs + name: host.os.kernel +- external: ecs + name: host.os.name +- external: ecs + name: host.os.platform +- external: ecs + name: host.os.version +- external: ecs + name: host.type diff --git a/packages/docker/data_stream/docker_logs/fields/fields.yml b/packages/docker/data_stream/docker_logs/fields/fields.yml new file mode 100644 index 00000000000..731024bef90 --- /dev/null +++ b/packages/docker/data_stream/docker_logs/fields/fields.yml @@ -0,0 +1,42 @@ +- name: docker.container.labels.* + type: object + release: ga + description: | + Container labels +- name: docker.container + type: group + release: ga + fields: + - name: command + type: keyword + description: | + Command that was executed in the Docker container. + - name: created + type: date + description: | + Date when the container was created. + - name: status + type: keyword + description: | + Container status. + - name: ip_addresses + type: ip + description: | + Container IP addresses. + - name: size + type: group + fields: + - name: root_fs + type: long + metric_type: gauge + description: | + Total size of all the files in the container. + - name: rw + type: long + metric_type: gauge + description: | + Size of the files that have been created or changed since creation. + - name: tags + type: keyword + description: | + Image tags. diff --git a/packages/docker/data_stream/docker_logs/manifest.yml b/packages/docker/data_stream/docker_logs/manifest.yml new file mode 100644 index 00000000000..30c85e08b6b --- /dev/null +++ b/packages/docker/data_stream/docker_logs/manifest.yml @@ -0,0 +1,56 @@ +title: "Docker container logs" +type: logs +streams: + - input: filestream + title: Collect Docker container logs + description: Collect Docker container logs + vars: + - name: paths + type: text + required: true + title: Docker container log path + multi: true + default: + - /var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log + - name: symlinks + type: bool + title: Use Symlinks + multi: false + required: true + show_user: true + default: true + - name: containerParserStream + type: text + title: Container parser's stream configuration + multi: false + required: true + default: all + - name: condition + title: Condition + description: Condition to filter when to apply this datastream + type: text + multi: false + required: false + show_user: true + - name: additionalParsersConfig + type: yaml + title: Additional parsers configuration + multi: false + required: true + default: | + # - ndjson: + # target: json + # ignore_decoding_error: true + # - multiline: + # type: pattern + # pattern: '^\[' + # negate: true + # match: after + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/docker/data_stream/docker_logs/sample_event.json b/packages/docker/data_stream/docker_logs/sample_event.json new file mode 100644 index 00000000000..947f24b84ff --- /dev/null +++ b/packages/docker/data_stream/docker_logs/sample_event.json @@ -0,0 +1,102 @@ +{ + "container": { + "image": { + "name": "docker.elastic.co/elastic-agent/elastic-agent-complete:8.5.0" + }, + "name": "elastic-package-stack_elastic-agent_1", + "id": "cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04", + "labels": { + "io_k8s_display-name": "Elastic-Agent image", + "org_opencontainers_image_title": "Elastic-Agent", + "com_docker_compose_oneoff": "False", + "release": "1", + "com_docker_compose_project": "elastic-package-stack", + "org_opencontainers_image_created": "2022-10-24T20:20:43Z", + "description": "Agent manages other beats based on configuration provided.", + "maintainer": "infra@elastic.co", + "org_opencontainers_image_vendor": "Elastic", + "org_label-schema_vcs-url": "github.com/elastic/elastic-agent", + "org_label-schema_vcs-ref": "9da6ba5fce5d6b4d2c473c1f5ff6056794e9a644", + "vendor": "Elastic", + "org_label-schema_vendor": "Elastic", + "com_docker_compose_service": "elastic-agent", + "org_opencontainers_image_licenses": "Elastic License", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "org_label-schema_license": "Elastic License", + "org_label-schema_build-date": "2022-10-24T20:20:43Z", + "summary": "elastic-agent", + "com_docker_compose_config-hash": "877e65101e9a2d525e764de557ab89ee529bee1f43d36e1f458fd3f9def52cf8", + "org_label-schema_version": "8.5.0", + "com_docker_compose_project_config_files": "/home/chrismark/.elastic-package/profiles/default/stack/snapshot.yml", + "version": "8.5.0", + "url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_name": "elastic-agent", + "license": "Elastic License", + "org_label-schema_schema-version": "1.0", + "name": "elastic-agent", + "com_docker_compose_container-number": "1", + "com_docker_compose_version": "1.29.2", + "com_docker_compose_project_working_dir": "/home/chrismark/.elastic-package/profiles/default/stack", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent" + } + }, + "agent": { + "name": "docker-fleet-agent", + "id": "069c0cc8-d191-42b2-92c8-fe4dd065685b", + "type": "filebeat", + "ephemeral_id": "93ca0744-1bef-4a2a-8534-6cbd9e33287a", + "version": "8.5.0" + }, + "log": { + "file": { + "path": "/var/lib/docker/containers/cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04/cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04-json.log" + }, + "offset": 17027 + }, + "elastic_agent": { + "id": "069c0cc8-d191-42b2-92c8-fe4dd065685b", + "version": "8.5.0", + "snapshot": false + }, + "message": "{\"log.level\":\"info\",\"@timestamp\":\"2022-11-24T10:16:39.493Z\",\"log.origin\":{\"file.name\":\"stateresolver/stateresolver.go\",\"file.line\":66},\"message\":\"Updating internal state\",\"ecs.version\":\"1.6.0\"}\n", + "input": { + "type": "filestream" + }, + "@timestamp": "2022-11-24T10:16:39.493Z", + "ecs": { + "version": "8.0.0" + }, + "stream": "stderr", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "docker.docker_logs" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.14.0-1054-oem", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.5 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": true, + "ip": [ + "172.26.0.7" + ], + "name": "docker-fleet-agent", + "id": "66392b0697b84641af8006d87aeb89f1", + "mac": [ + "02-42-AC-1A-00-07" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2022-11-24T10:16:42Z", + "dataset": "docker.docker_logs" + } +} \ No newline at end of file diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index f4bfa0be161..71a88270e91 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,8 +1,8 @@ name: docker -title: Docker Metrics -version: 2.2.0 +title: Docker Metrics & Logs +version: 2.3.0 release: ga -description: Collect metrics from Docker instances with Elastic Agent. +description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: - src: /img/logo_docker.svg @@ -29,5 +29,8 @@ policy_templates: - type: docker/metrics title: Collect Docker metrics description: Collecting container, cpu, diskio, event, healthcheck, image, info, memory and network metrics from Docker instances + - type: filestream + title: Collect Docker container logs + description: Collecting docker container logs owner: github: elastic/obs-cloudnative-monitoring From 83b84fc3400821011837b823ff8e056ad2b9743a Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 24 Nov 2022 12:53:17 +0200 Subject: [PATCH 2/6] fix the docs Signed-off-by: ChrsMark --- packages/docker/_dev/build/docs/README.md | 34 ++++- packages/docker/docs/README.md | 176 +++++++++++++++++++++- 2 files changed, 207 insertions(+), 3 deletions(-) diff --git a/packages/docker/_dev/build/docs/README.md b/packages/docker/_dev/build/docs/README.md index 4fcf11cce35..79f5f29b0a9 100644 --- a/packages/docker/_dev/build/docs/README.md +++ b/packages/docker/_dev/build/docs/README.md @@ -1,6 +1,9 @@ # Docker Integration -This Integration fetches metrics from [Docker](https://www.docker.com/) containers. The default data streams are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` and `network`. The `image` metricset is not enabled by default. +This Integration collects metrics and logs from [Docker](https://www.docker.com/) containers. +The default data streams for metrics collection are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` +and `network`. The `image` metricset is not enabled by default. +The `docker_logs` data stream for containers' logs collection is enabled by default. ## Compatibility @@ -22,6 +25,25 @@ docker run -d \ -E output.elasticsearch.hosts=["elasticsearch:9200"] ``` +For log collection since the discovery of the containers happen automatically, again access to `unix:///var/run/docker.sock` +will be needed so as Agent to be able to watch for Container events. +In addition, access is required to the containers' logs files which by default follows the pattern of +`/var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log` +If Elastic Agent is running inside docker, you'll need to mount the logs' directory too inside the container: + + +``` +docker run -d \ + --name=metricbeat \ + --user=root \ + --volume="/var/run/docker.sock:/var/run/docker.sock:ro" \ + --volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" \ + docker.elastic.co/beats/metricbeat:latest metricbeat -e \ + -E output.elasticsearch.hosts=["elasticsearch:9200"] +``` + +In all cases make sure that Agent has the proper permissions to access these files. + ## Module-specific configuration notes It is strongly recommended that you run Docker metricsets with a @@ -110,4 +132,12 @@ The Docker `network` data stream collects network metrics. {{fields "network"}} -{{event "network"}} \ No newline at end of file +{{event "network"}} + +### Docker_logs + +The Docker `docker_logs` data stream collects container logs. + +{{fields "docker_logs"}} + +{{event "docker_logs"}} \ No newline at end of file diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index fefa34071a7..a854b2b8a42 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -1,6 +1,9 @@ # Docker Integration -This Integration fetches metrics from [Docker](https://www.docker.com/) containers. The default data streams are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` and `network`. The `image` metricset is not enabled by default. +This Integration collects metrics and logs from [Docker](https://www.docker.com/) containers. +The default data streams for metrics collection are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` +and `network`. The `image` metricset is not enabled by default. +The `docker_logs` data stream for containers' logs collection is enabled by default. ## Compatibility @@ -22,6 +25,25 @@ docker run -d \ -E output.elasticsearch.hosts=["elasticsearch:9200"] ``` +For log collection since the discovery of the containers happen automatically, again access to `unix:///var/run/docker.sock` +will be needed so as Agent to be able to watch for Container events. +In addition, access is required to the containers' logs files which by default follows the pattern of +`/var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log` +If Elastic Agent is running inside docker, you'll need to mount the logs' directory too inside the container: + + +``` +docker run -d \ + --name=metricbeat \ + --user=root \ + --volume="/var/run/docker.sock:/var/run/docker.sock:ro" \ + --volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" \ + docker.elastic.co/beats/metricbeat:latest metricbeat -e \ + -E output.elasticsearch.hosts=["elasticsearch:9200"] +``` + +In all cases make sure that Agent has the proper permissions to access these files. + ## Module-specific configuration notes It is strongly recommended that you run Docker metricsets with a @@ -1043,4 +1065,156 @@ An example event for `network` looks as following: "type": "docker" } } +``` + +### Docker_logs + +The Docker `docker_logs` data stream collects container logs. + +**Exported fields** + +| Field | Description | Type | Metric Type | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| container.id | Unique container id. | keyword | | +| container.image.name | Name of the image the container was built on. | keyword | | +| container.name | Container name. | keyword | | +| container.runtime | Runtime managing this container. | keyword | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| docker.container.command | Command that was executed in the Docker container. | keyword | | +| docker.container.created | Date when the container was created. | date | | +| docker.container.ip_addresses | Container IP addresses. | ip | | +| docker.container.labels.\* | Container labels | object | | +| docker.container.size.root_fs | Total size of all the files in the container. | long | gauge | +| docker.container.size.rw | Size of the files that have been created or changed since creation. | long | gauge | +| docker.container.status | Container status. | keyword | | +| docker.container.tags | Image tags. | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | +| event.dataset | Event dataset | constant_keyword | | +| event.module | Event module | constant_keyword | | +| host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | +| host.architecture | Operating system architecture. | keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | +| host.os.full | Operating system name, including the version or code name. | keyword | | +| host.os.full.text | Multi-field of `host.os.full`. | match_only_text | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | +| host.os.name | Operating system name, without the version. | keyword | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | +| host.os.version | Operating system version as a raw string. | keyword | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | + + +An example event for `docker` looks as following: + +```json +{ + "container": { + "image": { + "name": "docker.elastic.co/elastic-agent/elastic-agent-complete:8.5.0" + }, + "name": "elastic-package-stack_elastic-agent_1", + "id": "cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04", + "labels": { + "io_k8s_display-name": "Elastic-Agent image", + "org_opencontainers_image_title": "Elastic-Agent", + "com_docker_compose_oneoff": "False", + "release": "1", + "com_docker_compose_project": "elastic-package-stack", + "org_opencontainers_image_created": "2022-10-24T20:20:43Z", + "description": "Agent manages other beats based on configuration provided.", + "maintainer": "infra@elastic.co", + "org_opencontainers_image_vendor": "Elastic", + "org_label-schema_vcs-url": "github.com/elastic/elastic-agent", + "org_label-schema_vcs-ref": "9da6ba5fce5d6b4d2c473c1f5ff6056794e9a644", + "vendor": "Elastic", + "org_label-schema_vendor": "Elastic", + "com_docker_compose_service": "elastic-agent", + "org_opencontainers_image_licenses": "Elastic License", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "org_label-schema_license": "Elastic License", + "org_label-schema_build-date": "2022-10-24T20:20:43Z", + "summary": "elastic-agent", + "com_docker_compose_config-hash": "877e65101e9a2d525e764de557ab89ee529bee1f43d36e1f458fd3f9def52cf8", + "org_label-schema_version": "8.5.0", + "com_docker_compose_project_config_files": "/home/chrismark/.elastic-package/profiles/default/stack/snapshot.yml", + "version": "8.5.0", + "url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_name": "elastic-agent", + "license": "Elastic License", + "org_label-schema_schema-version": "1.0", + "name": "elastic-agent", + "com_docker_compose_container-number": "1", + "com_docker_compose_version": "1.29.2", + "com_docker_compose_project_working_dir": "/home/chrismark/.elastic-package/profiles/default/stack", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent" + } + }, + "agent": { + "name": "docker-fleet-agent", + "id": "069c0cc8-d191-42b2-92c8-fe4dd065685b", + "type": "filebeat", + "ephemeral_id": "93ca0744-1bef-4a2a-8534-6cbd9e33287a", + "version": "8.5.0" + }, + "log": { + "file": { + "path": "/var/lib/docker/containers/cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04/cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04-json.log" + }, + "offset": 17027 + }, + "elastic_agent": { + "id": "069c0cc8-d191-42b2-92c8-fe4dd065685b", + "version": "8.5.0", + "snapshot": false + }, + "message": "{\"log.level\":\"info\",\"@timestamp\":\"2022-11-24T10:16:39.493Z\",\"log.origin\":{\"file.name\":\"stateresolver/stateresolver.go\",\"file.line\":66},\"message\":\"Updating internal state\",\"ecs.version\":\"1.6.0\"}\n", + "input": { + "type": "filestream" + }, + "@timestamp": "2022-11-24T10:16:39.493Z", + "ecs": { + "version": "8.0.0" + }, + "stream": "stderr", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "docker.docker_logs" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.14.0-1054-oem", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.5 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": true, + "ip": [ + "172.26.0.7" + ], + "name": "docker-fleet-agent", + "id": "66392b0697b84641af8006d87aeb89f1", + "mac": [ + "02-42-AC-1A-00-07" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2022-11-24T10:16:42Z", + "dataset": "docker.docker_logs" + } +} ``` \ No newline at end of file From 6588817f12b2e1ee13e689a9aba2e2e36d430ec6 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 24 Nov 2022 13:46:13 +0200 Subject: [PATCH 3/6] Add fields Signed-off-by: ChrsMark --- .../data_stream/docker_logs/fields/fields.yml | 73 +++++++++---------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/packages/docker/data_stream/docker_logs/fields/fields.yml b/packages/docker/data_stream/docker_logs/fields/fields.yml index 731024bef90..a74576e8479 100644 --- a/packages/docker/data_stream/docker_logs/fields/fields.yml +++ b/packages/docker/data_stream/docker_logs/fields/fields.yml @@ -1,42 +1,39 @@ -- name: docker.container.labels.* +- name: container.labels.* type: object release: ga description: | Container labels -- name: docker.container - type: group - release: ga - fields: - - name: command - type: keyword - description: | - Command that was executed in the Docker container. - - name: created - type: date - description: | - Date when the container was created. - - name: status - type: keyword - description: | - Container status. - - name: ip_addresses - type: ip - description: | - Container IP addresses. - - name: size - type: group - fields: - - name: root_fs - type: long - metric_type: gauge - description: | - Total size of all the files in the container. - - name: rw - type: long - metric_type: gauge - description: | - Size of the files that have been created or changed since creation. - - name: tags - type: keyword - description: | - Image tags. +- name: container.name + type: keyword + release: ga + description: Container name +- name: container.id + type: keyword + release: ga + description: Container ID +- name: container.image.name + type: keyword + release: ga + description: Container image name +- name: stream + type: keyword + release: ga + description: Container log stream +- name: message + type: keyword + release: ga + description: Container log message +- name: message + type: keyword + release: ga + description: Container log message +- name: input.type + type: keyword + release: ga + description: Input type +- name: log.offset + type: keyword + release: ga + description: Log offset + + From d384b42dfef3475eef130ae774dbe10fb1f13866 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Thu, 24 Nov 2022 13:58:21 +0200 Subject: [PATCH 4/6] Re build package Signed-off-by: ChrsMark --- .../docker_logs/fields/base-fields.yml | 17 +++++ .../data_stream/docker_logs/fields/fields.yml | 21 ------ packages/docker/docs/README.md | 72 +++++++++---------- 3 files changed, 52 insertions(+), 58 deletions(-) diff --git a/packages/docker/data_stream/docker_logs/fields/base-fields.yml b/packages/docker/data_stream/docker_logs/fields/base-fields.yml index c6fcca2192b..a49384ce82f 100644 --- a/packages/docker/data_stream/docker_logs/fields/base-fields.yml +++ b/packages/docker/data_stream/docker_logs/fields/base-fields.yml @@ -18,3 +18,20 @@ type: constant_keyword description: Event dataset value: docker.docker_logs +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + type: keyword + description: Path to the log file. +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: stream + type: keyword + release: ga + description: Container log stream +- name: message + type: keyword + release: ga + description: Container log message diff --git a/packages/docker/data_stream/docker_logs/fields/fields.yml b/packages/docker/data_stream/docker_logs/fields/fields.yml index a74576e8479..def70553fa0 100644 --- a/packages/docker/data_stream/docker_logs/fields/fields.yml +++ b/packages/docker/data_stream/docker_logs/fields/fields.yml @@ -15,25 +15,4 @@ type: keyword release: ga description: Container image name -- name: stream - type: keyword - release: ga - description: Container log stream -- name: message - type: keyword - release: ga - description: Container log message -- name: message - type: keyword - release: ga - description: Container log message -- name: input.type - type: keyword - release: ga - description: Input type -- name: log.offset - type: keyword - release: ga - description: Log offset - diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index a854b2b8a42..78867b7c05e 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -1073,43 +1073,41 @@ The Docker `docker_logs` data stream collects container logs. **Exported fields** -| Field | Description | Type | Metric Type | -|---|---|---|---| -| @timestamp | Event timestamp. | date | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.name | Container name. | keyword | | -| container.runtime | Runtime managing this container. | keyword | | -| data_stream.dataset | Data stream dataset. | constant_keyword | | -| data_stream.namespace | Data stream namespace. | constant_keyword | | -| data_stream.type | Data stream type. | constant_keyword | | -| docker.container.command | Command that was executed in the Docker container. | keyword | | -| docker.container.created | Date when the container was created. | date | | -| docker.container.ip_addresses | Container IP addresses. | ip | | -| docker.container.labels.\* | Container labels | object | | -| docker.container.size.root_fs | Total size of all the files in the container. | long | gauge | -| docker.container.size.rw | Size of the files that have been created or changed since creation. | long | gauge | -| docker.container.status | Container status. | keyword | | -| docker.container.tags | Image tags. | keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.dataset | Event dataset | constant_keyword | | -| event.module | Event module | constant_keyword | | -| host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | -| host.architecture | Operating system architecture. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.full | Operating system name, including the version or code name. | keyword | | -| host.os.full.text | Multi-field of `host.os.full`. | match_only_text | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| container.id | Container ID | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels.\* | Container labels | object | +| container.name | Container name. | keyword | +| container.runtime | Runtime managing this container. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | +| host.architecture | Operating system architecture. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.full | Operating system name, including the version or code name. | keyword | +| host.os.full.text | Multi-field of `host.os.full`. | match_only_text | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Path to the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| message | Container log message | keyword | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| stream | Container log stream | keyword | An example event for `docker` looks as following: From 192f4c0102e370c597027a0343ba6e2006e6f6ff Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Mon, 28 Nov 2022 10:21:01 +0200 Subject: [PATCH 5/6] Fix naming Signed-off-by: ChrsMark --- packages/docker/_dev/build/docs/README.md | 10 +++++----- packages/docker/changelog.yml | 4 ++-- .../agent/stream/stream.yml.hbs | 0 .../fields/base-fields.yml | 2 +- .../{docker_logs => container_logs}/fields/ecs.yml | 0 .../fields/fields.yml | 1 - .../{docker_logs => container_logs}/manifest.yml | 7 ------- .../sample_event.json | 4 ++-- packages/docker/docs/README.md | 12 ++++++------ 9 files changed, 16 insertions(+), 24 deletions(-) rename packages/docker/data_stream/{docker_logs => container_logs}/agent/stream/stream.yml.hbs (100%) rename packages/docker/data_stream/{docker_logs => container_logs}/fields/base-fields.yml (96%) rename packages/docker/data_stream/{docker_logs => container_logs}/fields/ecs.yml (100%) rename packages/docker/data_stream/{docker_logs => container_logs}/fields/fields.yml (99%) rename packages/docker/data_stream/{docker_logs => container_logs}/manifest.yml (91%) rename packages/docker/data_stream/{docker_logs => container_logs}/sample_event.json (97%) diff --git a/packages/docker/_dev/build/docs/README.md b/packages/docker/_dev/build/docs/README.md index 79f5f29b0a9..b54c1a40da0 100644 --- a/packages/docker/_dev/build/docs/README.md +++ b/packages/docker/_dev/build/docs/README.md @@ -3,7 +3,7 @@ This Integration collects metrics and logs from [Docker](https://www.docker.com/) containers. The default data streams for metrics collection are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` and `network`. The `image` metricset is not enabled by default. -The `docker_logs` data stream for containers' logs collection is enabled by default. +The `container_logs` data stream for containers' logs collection is enabled by default. ## Compatibility @@ -134,10 +134,10 @@ The Docker `network` data stream collects network metrics. {{event "network"}} -### Docker_logs +### container_logs -The Docker `docker_logs` data stream collects container logs. +The Docker `container_logs` data stream collects container logs. -{{fields "docker_logs"}} +{{fields "container_logs"}} -{{event "docker_logs"}} \ No newline at end of file +{{event "container_logs"}} \ No newline at end of file diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index 443436f751d..07c52b61621 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top - version: "2.3.0" changes: - - description: Add docker_logs + - description: Add container_logs type: enhancement - link: http://github.com/elastic/integrations/pull/9999 + link: http://github.com/elastic/integrations/pull/4716 - version: "2.2.0" changes: - description: Add skip_major flag diff --git a/packages/docker/data_stream/docker_logs/agent/stream/stream.yml.hbs b/packages/docker/data_stream/container_logs/agent/stream/stream.yml.hbs similarity index 100% rename from packages/docker/data_stream/docker_logs/agent/stream/stream.yml.hbs rename to packages/docker/data_stream/container_logs/agent/stream/stream.yml.hbs diff --git a/packages/docker/data_stream/docker_logs/fields/base-fields.yml b/packages/docker/data_stream/container_logs/fields/base-fields.yml similarity index 96% rename from packages/docker/data_stream/docker_logs/fields/base-fields.yml rename to packages/docker/data_stream/container_logs/fields/base-fields.yml index a49384ce82f..74e6bbb96d7 100644 --- a/packages/docker/data_stream/docker_logs/fields/base-fields.yml +++ b/packages/docker/data_stream/container_logs/fields/base-fields.yml @@ -17,7 +17,7 @@ - name: event.dataset type: constant_keyword description: Event dataset - value: docker.docker_logs + value: docker.container_logs - name: log.offset type: long description: Offset of the entry in the log file. diff --git a/packages/docker/data_stream/docker_logs/fields/ecs.yml b/packages/docker/data_stream/container_logs/fields/ecs.yml similarity index 100% rename from packages/docker/data_stream/docker_logs/fields/ecs.yml rename to packages/docker/data_stream/container_logs/fields/ecs.yml diff --git a/packages/docker/data_stream/docker_logs/fields/fields.yml b/packages/docker/data_stream/container_logs/fields/fields.yml similarity index 99% rename from packages/docker/data_stream/docker_logs/fields/fields.yml rename to packages/docker/data_stream/container_logs/fields/fields.yml index def70553fa0..41bc9f09e94 100644 --- a/packages/docker/data_stream/docker_logs/fields/fields.yml +++ b/packages/docker/data_stream/container_logs/fields/fields.yml @@ -15,4 +15,3 @@ type: keyword release: ga description: Container image name - diff --git a/packages/docker/data_stream/docker_logs/manifest.yml b/packages/docker/data_stream/container_logs/manifest.yml similarity index 91% rename from packages/docker/data_stream/docker_logs/manifest.yml rename to packages/docker/data_stream/container_logs/manifest.yml index 30c85e08b6b..497ed2788ec 100644 --- a/packages/docker/data_stream/docker_logs/manifest.yml +++ b/packages/docker/data_stream/container_logs/manifest.yml @@ -12,13 +12,6 @@ streams: multi: true default: - /var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log - - name: symlinks - type: bool - title: Use Symlinks - multi: false - required: true - show_user: true - default: true - name: containerParserStream type: text title: Container parser's stream configuration diff --git a/packages/docker/data_stream/docker_logs/sample_event.json b/packages/docker/data_stream/container_logs/sample_event.json similarity index 97% rename from packages/docker/data_stream/docker_logs/sample_event.json rename to packages/docker/data_stream/container_logs/sample_event.json index 947f24b84ff..e86037c261c 100644 --- a/packages/docker/data_stream/docker_logs/sample_event.json +++ b/packages/docker/data_stream/container_logs/sample_event.json @@ -70,7 +70,7 @@ "data_stream": { "namespace": "default", "type": "logs", - "dataset": "docker.docker_logs" + "dataset": "docker.container_logs" }, "host": { "hostname": "docker-fleet-agent", @@ -97,6 +97,6 @@ "event": { "agent_id_status": "verified", "ingested": "2022-11-24T10:16:42Z", - "dataset": "docker.docker_logs" + "dataset": "docker.container_logs" } } \ No newline at end of file diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index 78867b7c05e..62147818194 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -3,7 +3,7 @@ This Integration collects metrics and logs from [Docker](https://www.docker.com/) containers. The default data streams for metrics collection are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` and `network`. The `image` metricset is not enabled by default. -The `docker_logs` data stream for containers' logs collection is enabled by default. +The `container_logs` data stream for containers' logs collection is enabled by default. ## Compatibility @@ -1067,9 +1067,9 @@ An example event for `network` looks as following: } ``` -### Docker_logs +### container_logs -The Docker `docker_logs` data stream collects container logs. +The Docker `container_logs` data stream collects container logs. **Exported fields** @@ -1110,7 +1110,7 @@ The Docker `docker_logs` data stream collects container logs. | stream | Container log stream | keyword | -An example event for `docker` looks as following: +An example event for `container` looks as following: ```json { @@ -1185,7 +1185,7 @@ An example event for `docker` looks as following: "data_stream": { "namespace": "default", "type": "logs", - "dataset": "docker.docker_logs" + "dataset": "docker.container_logs" }, "host": { "hostname": "docker-fleet-agent", @@ -1212,7 +1212,7 @@ An example event for `docker` looks as following: "event": { "agent_id_status": "verified", "ingested": "2022-11-24T10:16:42Z", - "dataset": "docker.docker_logs" + "dataset": "docker.container_logs" } } ``` \ No newline at end of file From e59e68c55d98855637aff88d05b6e1bb8a25b67f Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Mon, 28 Nov 2022 11:38:13 +0200 Subject: [PATCH 6/6] Fix pattern to catch log rotation Signed-off-by: ChrsMark --- packages/docker/_dev/build/docs/README.md | 2 +- packages/docker/data_stream/container_logs/manifest.yml | 2 +- packages/docker/docs/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/docker/_dev/build/docs/README.md b/packages/docker/_dev/build/docs/README.md index b54c1a40da0..6ce694e33d7 100644 --- a/packages/docker/_dev/build/docs/README.md +++ b/packages/docker/_dev/build/docs/README.md @@ -28,7 +28,7 @@ docker run -d \ For log collection since the discovery of the containers happen automatically, again access to `unix:///var/run/docker.sock` will be needed so as Agent to be able to watch for Container events. In addition, access is required to the containers' logs files which by default follows the pattern of -`/var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log` +`/var/lib/docker/containers/${docker.container.id}/*-json.log` If Elastic Agent is running inside docker, you'll need to mount the logs' directory too inside the container: diff --git a/packages/docker/data_stream/container_logs/manifest.yml b/packages/docker/data_stream/container_logs/manifest.yml index 497ed2788ec..ac6e6514f1d 100644 --- a/packages/docker/data_stream/container_logs/manifest.yml +++ b/packages/docker/data_stream/container_logs/manifest.yml @@ -11,7 +11,7 @@ streams: title: Docker container log path multi: true default: - - /var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log + - /var/lib/docker/containers/${docker.container.id}/*-json.log - name: containerParserStream type: text title: Container parser's stream configuration diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index 62147818194..fc243f3d8bf 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -28,7 +28,7 @@ docker run -d \ For log collection since the discovery of the containers happen automatically, again access to `unix:///var/run/docker.sock` will be needed so as Agent to be able to watch for Container events. In addition, access is required to the containers' logs files which by default follows the pattern of -`/var/lib/docker/containers/${docker.container.id}/${docker.container.id}-json.log` +`/var/lib/docker/containers/${docker.container.id}/*-json.log` If Elastic Agent is running inside docker, you'll need to mount the logs' directory too inside the container: