Support additional parser configuration: ndjson and multiline in container logs data-stream

packages/kubernetes/_dev/build/docs/container-logs.md

@@ -3,4 +3,6 @@
 container-logs integration collects and parses logs of Kubernetes containers.
 
 It requires access to the log files in each Kubernetes node where the container logs are stored.
-This defaults to `/var/log/containers/*${kubernetes.container.id}.log`.
+This defaults to `/var/log/containers/*${kubernetes.container.id}.log`.
+
+By default only (container parser)[https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_parsers] is enabled. Additional log parsers can be added as an advanced options configuration.

packages/kubernetes/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.8.0"
+  changes:
+    - description: Support json logs parsing
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2345
 - version: "1.7.0"
   changes:
     - description: Add new audit logs data stream in kubernetes integration

packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs

@@ -4,4 +4,7 @@ paths:
 {{/each}}
 prospector.scanner.symlinks: {{ symlinks }}
 parsers:
-  - container: ~
+- container:
+    stream: {{ containerParserStream }}
+    format: {{ containerParserFormat }}
+{{ additionalParsersConfig }}

packages/kubernetes/data_stream/container_logs/fields/base-fields.yml

@@ -16,6 +16,9 @@
 - name: log.file.path
   type: keyword
   description: Path to the log file.
+- name: input.type
+  description: Type of Filebeat input.
+  type: keyword
 - name: kubernetes
   type: group
   fields:
@@ -51,6 +54,44 @@
       description: >
         Kubernetes hostname as reported by the node’s kernel
 
+    - name: node.labels.*
+      type: object
+      object_type: keyword
+      object_type_mapping_type: "*"
+      description: >
+        Kubernetes node labels map
+
+    - name: node.annotations.*
+      type: object
+      object_type: keyword
+      object_type_mapping_type: "*"
+      description: >
+        Kubernetes node annotations map
+
+    - name: node.uid
+      type: keyword
+      description: >
+        Kubernetes node UID
+
+    - name: namespace_uid
+      type: keyword
+      description: >
+        Kubernetes namespace UID
+
+    - name: namespace_labels.*
+      type: object
+      object_type: keyword
+      object_type_mapping_type: "*"
+      description: >
+        Kubernetes namespace labels map
+
+    - name: namespace_annotations.*
+      type: object
+      object_type: keyword
+      object_type_mapping_type: "*"
+      description: >
+        Kubernetes namespace annotations map
+
     - name: labels.*
       type: object
       object_type: keyword

packages/kubernetes/data_stream/container_logs/fields/ecs.yml

@@ -16,3 +16,9 @@
   name: agent.version
 - external: ecs
   name: message
+- external: ecs
+  name: container.runtime
+- external: ecs
+  name: orchestrator.cluster.name
+- external: ecs
+  name: orchestrator.cluster.url

packages/kubernetes/data_stream/container_logs/manifest.yml

@@ -19,3 +19,28 @@ streams:
         required: true
         show_user: true
         default: true
+      - name: containerParserStream
+        type: text
+        title: Container parser's stream configuration
+        multi: false
+        required: true
+        default: all
+      - name: containerParserFormat
+        type: text
+        title: Container parser's format configuration
+        multi: false
+        required: true
+        default: auto
+      - name: additionalParsersConfig
+        type: yaml
+        title: Additional parsers configuration
+        multi: false
+        required: true
+        default: |
+          # - ndjson:
+          #     target: json
+          # - multiline:
+          #     type: pattern
+          #     pattern: '^\['
+          #     negate: true
+          #     match: after

packages/kubernetes/data_stream/container_logs/sample_event.json

@@ -0,0 +1,113 @@
+{
+    "container": {
+        "image": {
+            "name": "nginx:1.14.2"
+        },
+        "runtime": "containerd",
+        "id": "6a5ac062689963aea9ee83f8e6adc2e1d658b280c0912e92c275a73c278ecd38"
+    },
+    "kubernetes": {
+        "container": {
+            "name": "nginx"
+        },
+        "node": {
+            "uid": "4b2a1961-1526-4ccb-bd8d-738dbbcf97da",
+            "hostname": "kind-control-plane",
+            "name": "kind-control-plane",
+            "labels": {
+                "node_kubernetes_io/exclude-from-external-load-balancers": "",
+                "node-role_kubernetes_io/master": "",
+                "kubernetes_io/hostname": "kind-control-plane",
+                "node-role_kubernetes_io/control-plane": "",
+                "beta_kubernetes_io/os": "linux",
+                "kubernetes_io/arch": "amd64",
+                "kubernetes_io/os": "linux",
+                "beta_kubernetes_io/arch": "amd64"
+            }
+        },
+        "pod": {
+            "uid": "bf2630e4-b6fa-4477-a6d0-ebf62d3ad495",
+            "ip": "10.244.0.10",
+            "name": "nginx-deployment-66b6c48dd5-ffdxp"
+        },
+        "namespace": "default",
+        "replicaset": {
+            "name": "nginx-deployment-66b6c48dd5"
+        },
+        "namespace_uid": "2774c099-c88d-4819-b87c-d0a6d7a3fc99",
+        "namespace_labels": {
+            "kubernetes_io/metadata_name": "default"
+        },
+        "deployment": {
+            "name": "nginx-deployment"
+        },
+        "labels": {
+            "app": "nginx",
+            "pod-template-hash": "66b6c48dd5"
+        }
+    },
+    "agent": {
+        "name": "kind-control-plane",
+        "id": "ae6e5950-8f6c-44a2-a801-1f8a21129d53",
+        "type": "filebeat",
+        "ephemeral_id": "fa7a4f61-3c25-43af-8765-350c5d7be20b",
+        "version": "8.1.0"
+    },
+    "log": {
+        "file": {
+            "path": "/var/log/containers/nginx-deployment-66b6c48dd5-ffdxp_default_nginx-6a5ac062689963aea9ee83f8e6adc2e1d658b280c0912e92c275a73c278ecd38.log"
+        },
+        "offset": 3673
+    },
+    "elastic_agent": {
+        "id": "ae6e5950-8f6c-44a2-a801-1f8a21129d53",
+        "version": "8.1.0",
+        "snapshot": true
+    },
+    "message": "127.0.0.1 - - [14/Dec/2021:09:42:30 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:95.0) Gecko/20100101 Firefox/95.0\" \"-\"",
+    "orchestrator": {
+        "cluster": {
+            "name": "kind",
+            "url": "kind-control-plane:6443"
+        }
+    },
+    "input": {
+        "type": "filestream"
+    },
+    "@timestamp": "2021-12-14T09:42:30.686Z",
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "data_stream": {
+        "namespace": "default",
+        "type": "logs",
+        "dataset": "kubernetes.container_logs"
+    },
+    "host": {
+        "hostname": "kind-control-plane",
+        "os": {
+            "kernel": "5.10.47-linuxkit",
+            "codename": "Core",
+            "name": "CentOS Linux",
+            "family": "redhat",
+            "type": "linux",
+            "version": "7 (Core)",
+            "platform": "centos"
+        },
+        "ip": [
+            "10.244.0.1"
+        ],
+        "containerized": true,
+        "name": "kind-control-plane",
+        "id": "f4e2f4a6efe0567a6719dc21d5d05a04",
+        "mac": [
+            "c6:7a:a1:3b:4b:43"
+        ],
+        "architecture": "x86_64"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "ingested": "2021-12-14T09:42:33Z",
+        "dataset": "kubernetes.container_logs"
+    }
+}

packages/kubernetes/docs/container-logs.md

@@ -3,4 +3,6 @@
 container-logs integration collects and parses logs of Kubernetes containers.
 
 It requires access to the log files in each Kubernetes node where the container logs are stored.
-This defaults to `/var/log/containers/*${kubernetes.container.id}.log`.
+This defaults to `/var/log/containers/*${kubernetes.container.id}.log`.
+
+By default only (container parser)[https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_parsers] is enabled. Additional log parsers can be added as an advanced options configuration.

packages/kubernetes/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: kubernetes
 title: Kubernetes
-version: 1.7.0
+version: 1.8.0
 license: basic
 description: Collect logs and metrics from Kubernetes clusters with Elastic Agent.
 type: integration