From 1dbbe4eedb90d632139f2c6df7b821fc2169580a Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Fri, 17 Dec 2021 13:20:58 +0100 Subject: [PATCH] Support additional parser configuration: ndjson and multiline in container logs data-stream (#2345) * add sample_event; support ndjson parser Signed-off-by: Tetiana Kravchenko * add possibility to adjust container, ndjson and multiline parser configuratios Signed-off-by: Tetiana Kravchenko * add pr link Signed-off-by: Tetiana Kravchenko * remove test; add container parser configuration; use generic Additional parsers configuration instead of multilineParser ad jsonParser Signed-off-by: Tetiana Kravchenko * add a link to parsers documentation Signed-off-by: Tetiana Kravchenko * bump package version to 1.8.0; add node.annotations. and namespace_annotations.* fields Signed-off-by: Tetiana Kravchenko * add container parser link to the documentation Signed-off-by: Tetiana Kravchenko * add multiline configuration example Signed-off-by: Tetiana Kravchenko --- .../_dev/build/docs/container-logs.md | 4 +- packages/kubernetes/changelog.yml | 5 + .../agent/stream/stream.yml.hbs | 5 +- .../container_logs/fields/base-fields.yml | 41 +++++++ .../data_stream/container_logs/fields/ecs.yml | 6 + .../data_stream/container_logs/manifest.yml | 25 ++++ .../container_logs/sample_event.json | 113 ++++++++++++++++++ packages/kubernetes/docs/container-logs.md | 4 +- packages/kubernetes/manifest.yml | 2 +- 9 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 packages/kubernetes/data_stream/container_logs/sample_event.json diff --git a/packages/kubernetes/_dev/build/docs/container-logs.md b/packages/kubernetes/_dev/build/docs/container-logs.md index 3595e1c2682..fa8fdb0728b 100644 --- a/packages/kubernetes/_dev/build/docs/container-logs.md +++ b/packages/kubernetes/_dev/build/docs/container-logs.md @@ -3,4 +3,6 @@ container-logs integration collects and parses logs of Kubernetes containers. It requires access to the log files in each Kubernetes node where the container logs are stored. -This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. \ No newline at end of file +This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. + +By default only (container parser)[https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_parsers] is enabled. Additional log parsers can be added as an advanced options configuration. diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index ba9ab79c47f..6cf930673ef 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.0" + changes: + - description: Support json logs parsing + type: enhancement + link: https://github.com/elastic/integrations/pull/2345 - version: "1.7.0" changes: - description: Add new audit logs data stream in kubernetes integration diff --git a/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs b/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs index 9432fd0a1af..c207e3fedc3 100644 --- a/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs +++ b/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs @@ -4,4 +4,7 @@ paths: {{/each}} prospector.scanner.symlinks: {{ symlinks }} parsers: - - container: ~ +- container: + stream: {{ containerParserStream }} + format: {{ containerParserFormat }} +{{ additionalParsersConfig }} diff --git a/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml b/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml index 21f9fc16fba..16f9c2a196e 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml @@ -16,6 +16,9 @@ - name: log.file.path type: keyword description: Path to the log file. +- name: input.type + description: Type of Filebeat input. + type: keyword - name: kubernetes type: group fields: @@ -51,6 +54,44 @@ description: > Kubernetes hostname as reported by the node’s kernel + - name: node.labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes node labels map + + - name: node.annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes node annotations map + + - name: node.uid + type: keyword + description: > + Kubernetes node UID + + - name: namespace_uid + type: keyword + description: > + Kubernetes namespace UID + + - name: namespace_labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes namespace labels map + + - name: namespace_annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes namespace annotations map + - name: labels.* type: object object_type: keyword diff --git a/packages/kubernetes/data_stream/container_logs/fields/ecs.yml b/packages/kubernetes/data_stream/container_logs/fields/ecs.yml index f6818be260a..32f8a586e74 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/ecs.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/ecs.yml @@ -16,3 +16,9 @@ name: agent.version - external: ecs name: message +- external: ecs + name: container.runtime +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/packages/kubernetes/data_stream/container_logs/manifest.yml b/packages/kubernetes/data_stream/container_logs/manifest.yml index 19bf942d940..581420a4729 100644 --- a/packages/kubernetes/data_stream/container_logs/manifest.yml +++ b/packages/kubernetes/data_stream/container_logs/manifest.yml @@ -19,3 +19,28 @@ streams: required: true show_user: true default: true + - name: containerParserStream + type: text + title: Container parser's stream configuration + multi: false + required: true + default: all + - name: containerParserFormat + type: text + title: Container parser's format configuration + multi: false + required: true + default: auto + - name: additionalParsersConfig + type: yaml + title: Additional parsers configuration + multi: false + required: true + default: | + # - ndjson: + # target: json + # - multiline: + # type: pattern + # pattern: '^\[' + # negate: true + # match: after diff --git a/packages/kubernetes/data_stream/container_logs/sample_event.json b/packages/kubernetes/data_stream/container_logs/sample_event.json new file mode 100644 index 00000000000..ee705f2d356 --- /dev/null +++ b/packages/kubernetes/data_stream/container_logs/sample_event.json @@ -0,0 +1,113 @@ +{ + "container": { + "image": { + "name": "nginx:1.14.2" + }, + "runtime": "containerd", + "id": "6a5ac062689963aea9ee83f8e6adc2e1d658b280c0912e92c275a73c278ecd38" + }, + "kubernetes": { + "container": { + "name": "nginx" + }, + "node": { + "uid": "4b2a1961-1526-4ccb-bd8d-738dbbcf97da", + "hostname": "kind-control-plane", + "name": "kind-control-plane", + "labels": { + "node_kubernetes_io/exclude-from-external-load-balancers": "", + "node-role_kubernetes_io/master": "", + "kubernetes_io/hostname": "kind-control-plane", + "node-role_kubernetes_io/control-plane": "", + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "amd64", + "kubernetes_io/os": "linux", + "beta_kubernetes_io/arch": "amd64" + } + }, + "pod": { + "uid": "bf2630e4-b6fa-4477-a6d0-ebf62d3ad495", + "ip": "10.244.0.10", + "name": "nginx-deployment-66b6c48dd5-ffdxp" + }, + "namespace": "default", + "replicaset": { + "name": "nginx-deployment-66b6c48dd5" + }, + "namespace_uid": "2774c099-c88d-4819-b87c-d0a6d7a3fc99", + "namespace_labels": { + "kubernetes_io/metadata_name": "default" + }, + "deployment": { + "name": "nginx-deployment" + }, + "labels": { + "app": "nginx", + "pod-template-hash": "66b6c48dd5" + } + }, + "agent": { + "name": "kind-control-plane", + "id": "ae6e5950-8f6c-44a2-a801-1f8a21129d53", + "type": "filebeat", + "ephemeral_id": "fa7a4f61-3c25-43af-8765-350c5d7be20b", + "version": "8.1.0" + }, + "log": { + "file": { + "path": "/var/log/containers/nginx-deployment-66b6c48dd5-ffdxp_default_nginx-6a5ac062689963aea9ee83f8e6adc2e1d658b280c0912e92c275a73c278ecd38.log" + }, + "offset": 3673 + }, + "elastic_agent": { + "id": "ae6e5950-8f6c-44a2-a801-1f8a21129d53", + "version": "8.1.0", + "snapshot": true + }, + "message": "127.0.0.1 - - [14/Dec/2021:09:42:30 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:95.0) Gecko/20100101 Firefox/95.0\" \"-\"", + "orchestrator": { + "cluster": { + "name": "kind", + "url": "kind-control-plane:6443" + } + }, + "input": { + "type": "filestream" + }, + "@timestamp": "2021-12-14T09:42:30.686Z", + "ecs": { + "version": "8.0.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "kubernetes.container_logs" + }, + "host": { + "hostname": "kind-control-plane", + "os": { + "kernel": "5.10.47-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "type": "linux", + "version": "7 (Core)", + "platform": "centos" + }, + "ip": [ + "10.244.0.1" + ], + "containerized": true, + "name": "kind-control-plane", + "id": "f4e2f4a6efe0567a6719dc21d5d05a04", + "mac": [ + "c6:7a:a1:3b:4b:43" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2021-12-14T09:42:33Z", + "dataset": "kubernetes.container_logs" + } +} \ No newline at end of file diff --git a/packages/kubernetes/docs/container-logs.md b/packages/kubernetes/docs/container-logs.md index 3595e1c2682..fa8fdb0728b 100644 --- a/packages/kubernetes/docs/container-logs.md +++ b/packages/kubernetes/docs/container-logs.md @@ -3,4 +3,6 @@ container-logs integration collects and parses logs of Kubernetes containers. It requires access to the log files in each Kubernetes node where the container logs are stored. -This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. \ No newline at end of file +This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. + +By default only (container parser)[https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_parsers] is enabled. Additional log parsers can be added as an advanced options configuration. diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index 7a34ca217dc..388c8bb03a6 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kubernetes title: Kubernetes -version: 1.7.0 +version: 1.8.0 license: basic description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration