Add docker_logs datastream for docker container logs collection (#4716)

packages/docker/_dev/build/docs/README.md

@@ -1,6 +1,9 @@
 # Docker Integration
 
-This Integration fetches metrics from [Docker](https://www.docker.com/) containers. The default data streams are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` and `network`. The `image` metricset is not enabled by default.
+This Integration collects metrics and logs from [Docker](https://www.docker.com/) containers. 
+The default data streams for metrics collection are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory`
+and `network`. The `image` metricset is not enabled by default.
+The `container_logs` data stream for containers' logs collection is enabled by default.
 
 ## Compatibility
 
@@ -22,6 +25,25 @@ docker run -d \
   -E output.elasticsearch.hosts=["elasticsearch:9200"]
 ```
 
+For log collection since the discovery of the containers happen automatically, again access to `unix:///var/run/docker.sock`
+will be needed so as Agent to be able to watch for Container events.
+In addition, access is required to the containers' logs files which by default follows the pattern of
+`/var/lib/docker/containers/${docker.container.id}/*-json.log`
+If Elastic Agent is running inside docker, you'll need to mount the logs' directory too inside the container:
+
+
+```
+docker run -d \
+  --name=metricbeat \
+  --user=root \
+  --volume="/var/run/docker.sock:/var/run/docker.sock:ro" \
+  --volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" \
+  docker.elastic.co/beats/metricbeat:latest metricbeat -e \
+  -E output.elasticsearch.hosts=["elasticsearch:9200"]
+```
+
+In all cases make sure that Agent has the proper permissions to access these files.
+
 ## Module-specific configuration notes
 
 It is strongly recommended that you run Docker metricsets with a
@@ -110,4 +132,12 @@ The Docker `network` data stream collects network metrics.
 
 {{fields "network"}}
 
-{{event "network"}}
+{{event "network"}}
+
+### container_logs
+
+The Docker `container_logs` data stream collects container logs.
+
+{{fields "container_logs"}}
+
+{{event "container_logs"}}

packages/docker/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.0"
+  changes:
+    - description: Add container_logs
+      type: enhancement
+      link: http://github.com/elastic/integrations/pull/4716
 - version: "2.2.0"
   changes:
     - description: Add skip_major flag

packages/docker/data_stream/container_logs/agent/stream/stream.yml.hbs

@@ -0,0 +1,18 @@
+id: docker-container-logs-${docker.container.name}-${docker.container.id}
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
+parsers:
+- container:
+    stream: {{ containerParserStream }}
+    format: docker
+{{ additionalParsersConfig }}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/docker/data_stream/container_logs/fields/base-fields.yml

@@ -0,0 +1,37 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: docker
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: docker.container_logs
+- name: log.offset
+  type: long
+  description: Offset of the entry in the log file.
+- name: log.file.path
+  type: keyword
+  description: Path to the log file.
+- name: input.type
+  description: Type of Filebeat input.
+  type: keyword
+- name: stream
+  type: keyword
+  release: ga
+  description: Container log stream
+- name: message
+  type: keyword
+  release: ga
+  description: Container log message

packages/docker/data_stream/container_logs/fields/ecs.yml

@@ -0,0 +1,38 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: service.address
+- external: ecs
+  name: service.type
+- external: ecs
+  name: container.id
+- external: ecs
+  name: container.name
+- external: ecs
+  name: container.runtime
+- external: ecs
+  name: container.image.name
+- external: ecs
+  name: host
+- external: ecs
+  name: host.architecture
+- external: ecs
+  name: host.ip
+- external: ecs
+  name: host.mac
+- external: ecs
+  name: host.name
+- external: ecs
+  name: host.os.family
+- external: ecs
+  name: host.os.full
+- external: ecs
+  name: host.os.kernel
+- external: ecs
+  name: host.os.name
+- external: ecs
+  name: host.os.platform
+- external: ecs
+  name: host.os.version
+- external: ecs
+  name: host.type

packages/docker/data_stream/container_logs/fields/fields.yml

@@ -0,0 +1,17 @@
+- name: container.labels.*
+  type: object
+  release: ga
+  description: |
+    Container labels
+- name: container.name
+  type: keyword
+  release: ga
+  description: Container name
+- name: container.id
+  type: keyword
+  release: ga
+  description: Container ID
+- name: container.image.name
+  type: keyword
+  release: ga
+  description: Container image name

packages/docker/data_stream/container_logs/manifest.yml

@@ -0,0 +1,49 @@
+title: "Docker container logs"
+type: logs
+streams:
+  - input: filestream
+    title: Collect Docker container logs
+    description: Collect Docker container logs
+    vars:
+      - name: paths
+        type: text
+        required: true
+        title: Docker container log path
+        multi: true
+        default:
+          - /var/lib/docker/containers/${docker.container.id}/*-json.log
+      - name: containerParserStream
+        type: text
+        title: Container parser's stream configuration
+        multi: false
+        required: true
+        default: all
+      - name: condition
+        title: Condition
+        description: Condition to filter when to apply this datastream
+        type: text
+        multi: false
+        required: false
+        show_user: true
+      - name: additionalParsersConfig
+        type: yaml
+        title: Additional parsers configuration
+        multi: false
+        required: true
+        default: |
+          # - ndjson:
+          #     target: json
+          #     ignore_decoding_error: true
+          # - multiline:
+          #     type: pattern
+          #     pattern: '^\['
+          #     negate: true
+          #     match: after
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.

packages/docker/data_stream/container_logs/sample_event.json

@@ -0,0 +1,102 @@
+{
+    "container": {
+        "image": {
+            "name": "docker.elastic.co/elastic-agent/elastic-agent-complete:8.5.0"
+        },
+        "name": "elastic-package-stack_elastic-agent_1",
+        "id": "cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04",
+        "labels": {
+            "io_k8s_display-name": "Elastic-Agent image",
+            "org_opencontainers_image_title": "Elastic-Agent",
+            "com_docker_compose_oneoff": "False",
+            "release": "1",
+            "com_docker_compose_project": "elastic-package-stack",
+            "org_opencontainers_image_created": "2022-10-24T20:20:43Z",
+            "description": "Agent manages other beats based on configuration provided.",
+            "maintainer": "[email protected]",
+            "org_opencontainers_image_vendor": "Elastic",
+            "org_label-schema_vcs-url": "github.com/elastic/elastic-agent",
+            "org_label-schema_vcs-ref": "9da6ba5fce5d6b4d2c473c1f5ff6056794e9a644",
+            "vendor": "Elastic",
+            "org_label-schema_vendor": "Elastic",
+            "com_docker_compose_service": "elastic-agent",
+            "org_opencontainers_image_licenses": "Elastic License",
+            "io_k8s_description": "Agent manages other beats based on configuration provided.",
+            "org_label-schema_license": "Elastic License",
+            "org_label-schema_build-date": "2022-10-24T20:20:43Z",
+            "summary": "elastic-agent",
+            "com_docker_compose_config-hash": "877e65101e9a2d525e764de557ab89ee529bee1f43d36e1f458fd3f9def52cf8",
+            "org_label-schema_version": "8.5.0",
+            "com_docker_compose_project_config_files": "/home/chrismark/.elastic-package/profiles/default/stack/snapshot.yml",
+            "version": "8.5.0",
+            "url": "https://www.elastic.co/beats/elastic-agent",
+            "org_label-schema_name": "elastic-agent",
+            "license": "Elastic License",
+            "org_label-schema_schema-version": "1.0",
+            "name": "elastic-agent",
+            "com_docker_compose_container-number": "1",
+            "com_docker_compose_version": "1.29.2",
+            "com_docker_compose_project_working_dir": "/home/chrismark/.elastic-package/profiles/default/stack",
+            "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent"
+        }
+    },
+    "agent": {
+        "name": "docker-fleet-agent",
+        "id": "069c0cc8-d191-42b2-92c8-fe4dd065685b",
+        "type": "filebeat",
+        "ephemeral_id": "93ca0744-1bef-4a2a-8534-6cbd9e33287a",
+        "version": "8.5.0"
+    },
+    "log": {
+        "file": {
+            "path": "/var/lib/docker/containers/cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04/cf67fae3321ec426e720311c345c758d5ceb5260e6ea171ea9ca509175458b04-json.log"
+        },
+        "offset": 17027
+    },
+    "elastic_agent": {
+        "id": "069c0cc8-d191-42b2-92c8-fe4dd065685b",
+        "version": "8.5.0",
+        "snapshot": false
+    },
+    "message": "{\"log.level\":\"info\",\"@timestamp\":\"2022-11-24T10:16:39.493Z\",\"log.origin\":{\"file.name\":\"stateresolver/stateresolver.go\",\"file.line\":66},\"message\":\"Updating internal state\",\"ecs.version\":\"1.6.0\"}\n",
+    "input": {
+        "type": "filestream"
+    },
+    "@timestamp": "2022-11-24T10:16:39.493Z",
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "stream": "stderr",
+    "data_stream": {
+        "namespace": "default",
+        "type": "logs",
+        "dataset": "docker.container_logs"
+    },
+    "host": {
+        "hostname": "docker-fleet-agent",
+        "os": {
+            "kernel": "5.14.0-1054-oem",
+            "codename": "focal",
+            "name": "Ubuntu",
+            "type": "linux",
+            "family": "debian",
+            "version": "20.04.5 LTS (Focal Fossa)",
+            "platform": "ubuntu"
+        },
+        "containerized": true,
+        "ip": [
+            "172.26.0.7"
+        ],
+        "name": "docker-fleet-agent",
+        "id": "66392b0697b84641af8006d87aeb89f1",
+        "mac": [
+            "02-42-AC-1A-00-07"
+        ],
+        "architecture": "x86_64"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "ingested": "2022-11-24T10:16:42Z",
+        "dataset": "docker.container_logs"
+    }
+}