Merge pull request #4 from observIQ/openshift

Create openshift plugin

plugins/openshift.yaml

@@ -0,0 +1,125 @@
+version: 0.0.1
+title: Openshift
+description: Log parser for Openshift
+parameters:
+  journald_log_path:
+    label: Containers Log Path
+    description: The directory where the journald logs are located. Defaults to '/var/log/journal'.
+    type: string
+  start_at:
+    label: Start At
+    description: Start reading file from 'beginning' or 'end'. Defaults to 'end'.
+    type: string
+  container_log:
+    label: Enable Container Logs
+    description: Enable collection of container logs
+    type: bool
+  docker_log:
+    label: Enable Docker Daemon Logs
+    description: Enable collection of the Docker Daemon logs
+    type: bool
+  openshift_log:
+    label: Enable Openshift logs
+    description: Enable collection of Openshift unit logs
+    type: bool
+
+pipeline:
+  - type: journald_input
+    directory: {{ or .journald_log_path "/var/log/journal" }}
+    start_at: {{ or .start_at "end" }}
+
+  - type: router
+    routes:
+      # Ignore logs written by Carbon to avoid circular parsing
+      - expr: '$record._SYSTEMD_UNIT == "docker.service" and $record.CONTAINER_NAME != nil and $record.CONTAINER_NAME matches "carbon"'
+        output: drop_output
+      # Send all container logs to the container name parser
+      - expr: '$record._SYSTEMD_UNIT == "docker.service" and $record.CONTAINER_NAME != nil'
+        output: {{ if .container_log -}} regex_parser {{- else -}} drop_output {{- end }}
+      # Send all docker logs to the be labeled
+      - expr: '$record._SYSTEMD_UNIT == "docker.service"'
+        output: {{ if .docker_log -}} docker_decorator {{- else -}} drop_output {{- end }}
+      # Send all openshift logs to be labeled and parsed
+      - expr: '$record._SYSTEMD_UNIT == "atomic-openshift-node.service"'
+        output: {{ if .openshift_log -}} openshift_decorator {{- else -}} drop_output {{- end }}
+
+  # Drop unwanted logs
+  - type: "drop_output"
+
+  # Parse the container name into service name, pod name, and namespace
+  - type: regex_parser
+    regex: '^(?P<service_name>[-a-z0-9_]+)_(?P<pod_name>[-a-z0-9]+)_(?P<namespace>[-a-z0-9]+)_[-a-z0-9]+_\d+$'
+    parse_from: CONTAINER_NAME
+    parse_to: k8s_metadata
+
+  # Move important fields to labels, and drop extraneous fields
+  - id: post_parse_restructure
+    type: restructure
+    ops:
+      - move:
+          from: k8s_metadata.namespace
+          to: $labels.namespace
+      - move:
+          from: k8s_metadata.pod_name
+          to: $labels.pod_name
+      - move:
+          from: k8s_metadata.service_name
+          to: $labels.service_name
+      - move:
+          from: _HOSTNAME
+          to: $labels.node_hostname
+      - move:
+          from: MESSAGE
+          to: $record
+      - add:
+          field: $labels.log_name
+          value_expr: '$labels.service_name'
+
+  # Decorate entries with k8s metadata
+  - type: k8s_metadata_decorator
+    pod_name_field: $labels.pod_name
+    namespace_field: $labels.namespace
+    output: {{ .output }}
+
+  # Add log_name to docker logs
+  - id: docker_decorator
+    type: metadata
+    labels:
+      log_name: 'docker'
+    output: {{ .output }}
+
+  # Add log_name to openshift logs
+  - id: openshift_decorator
+    type: metadata
+    labels:
+      log_name: 'openshift'
+    output: message_retainer
+
+  # For openshift logs, promote hostname to labels and remove all but message
+  - id: message_retainer
+    type: restructure
+    ops:
+      - move:
+          from: _HOSTNAME
+          to: $labels.hostname
+      - move:
+          from: MESSAGE
+          to: $record
+    output: message_regex_parser
+
+  # Parse the message
+  - id: message_regex_parser
+    type: regex_parser
+    regex: '(?P<severity>\w)(?P<timestamp>\d{4} \d{2}:\d{2}:\d{2}.\d+)\s+(?P<pid>\d+)\s+(?P<source>[^ \]]+)\] (?P<message>.*)'
+    severity:
+      parse_from: severity
+      mapping:
+        debug: d
+        info: i
+        warning: w
+        error: e
+        critical: c
+    timestamp:
+      parse_from: timestamp
+      layout: '%m%d %H:%M:%S.%s'
+    output: {{ .output }}