File based fluentd-daemonset-file

docker-image/v0.12/alpine-file/.dockerignore

@@ -0,0 +1 @@
+**/*.gitkeep

docker-image/v0.12/alpine-file/Dockerfile

@@ -0,0 +1,18 @@
+FROM fluent/fluentd:v0.12.33
+USER root
+WORKDIR /home/fluent
+
+# Copy configuration files
+COPY ./conf/fluent.conf /fluentd/etc/
+COPY ./conf/kubernetes.conf /fluentd/etc/
+
+# Copy plugins
+COPY plugins /fluentd/plugins/
+COPY entrypoint.sh /fluentd/entrypoint.sh
+
+# Environment variables
+ENV FLUENTD_OPT=""
+ENV FLUENTD_CONF="fluent.conf"
+
+# Run Fluentd
+CMD ["/fluentd/entrypoint.sh"]

docker-image/v0.12/alpine-file/conf/fluent.conf

@@ -0,0 +1,15 @@
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
+
+@include kubernetes.conf
+
+<match **>
+  @type file
+  path /var/log/fluent/
+  time_slice_format %Y%m%d
+  time_slice_wait 10m
+  time_format %Y%m%dT%H%M%S%z
+  compress gzip
+  utc
+</match>

docker-image/v0.12/alpine-file/conf/kubernetes.conf

@@ -0,0 +1,142 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb
+
+<match fluent.**>
+  type null
+</match>
+
+<source>
+  type tail
+  path /var/log/containers/*.log
+  pos_file /var/log/fluentd-containers.log.pos
+  time_format %Y-%m-%dT%H:%M:%S.%NZ
+  tag kubernetes.*
+  format json
+  read_from_head true
+</source>
+
+<source>
+  type tail
+  format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
+  time_format %Y-%m-%d %H:%M:%S
+  path /var/log/salt/minion
+  pos_file /var/log/fluentd-salt.pos
+  tag salt
+</source>
+
+<source>
+  type tail
+  format syslog
+  path /var/log/startupscript.log
+  pos_file /var/log/fluentd-startupscript.log.pos
+  tag startupscript
+</source>
+
+<source>
+  type tail
+  format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+  path /var/log/docker.log
+  pos_file /var/log/fluentd-docker.log.pos
+  tag docker
+</source>
+
+<source>
+  type tail
+  format none
+  path /var/log/etcd.log
+  pos_file /var/log/fluentd-etcd.log.pos
+  tag etcd
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/kubelet.log
+  pos_file /var/log/fluentd-kubelet.log.pos
+  tag kubelet
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/kube-proxy.log
+  pos_file /var/log/fluentd-kube-proxy.log.pos
+  tag kube-proxy
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/kube-apiserver.log
+  pos_file /var/log/fluentd-kube-apiserver.log.pos
+  tag kube-apiserver
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/kube-controller-manager.log
+  pos_file /var/log/fluentd-kube-controller-manager.log.pos
+  tag kube-controller-manager
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/kube-scheduler.log
+  pos_file /var/log/fluentd-kube-scheduler.log.pos
+  tag kube-scheduler
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/rescheduler.log
+  pos_file /var/log/fluentd-rescheduler.log.pos
+  tag rescheduler
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/glbc.log
+  pos_file /var/log/fluentd-glbc.log.pos
+  tag glbc
+</source>
+
+<source>
+  type tail
+  format kubernetes
+  multiline_flush_interval 5s
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/fluentd-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+</source>
+
+# Example:
+# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
+# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\S+\s+AUDIT:/
+  # Fields must be explicitly captured by name to be parsed into the record.
+  # Fields may not always be present, and order may change, so this just looks
+  # for a list of key="\"quoted\" value" pairs separated by spaces.
+  # Unknown fields are ignored.
+  # Note: We can't separate query/response lines as format1/format2 because
+  #       they don't always come one after the other for a given query.
+  format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
+  time_format %FT%T.%L%Z
+  path /var/log/kubernetes/kube-apiserver-audit.log
+  pos_file /var/log/kube-apiserver-audit.log.pos
+  tag kube-apiserver-audit
+</source>

docker-image/v0.12/alpine-file/conf/systemd.conf

@@ -0,0 +1,31 @@
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb
+
+# Logs from systemd-journal for interesting services.
+<source>
+  @type systemd
+  filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+  pos_file /var/log/fluentd-journald-kubelet.pos
+  read_from_head true
+  tag kubelet
+</source>
+
+# Logs from docker-systemd
+<source>
+  @type systemd
+  filters [{ "_SYSTEMD_UNIT": "docker.service" }]
+  pos_file /var/log/fluentd-journald-docker.pos
+  read_from_head true
+  tag docker.systemd
+</source>
+
+# Logs from systemd-journal for interesting services.
+<source>
+  @type systemd
+  filters [{ "_SYSTEMD_UNIT": "bootkube.service" }]
+  pos_file /var/log/fluentd-journald-bootkube.pos
+  read_from_head true
+  tag bootkube
+</source>
+

docker-image/v0.12/alpine-file/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+if [ -z ${FLUENT_ELASTICSEARCH_USER} ] ; then
+   sed -i  '/FLUENT_ELASTICSEARCH_USER/d' /fluentd/etc/${FLUENTD_CONF}
+fi
+
+if [ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ] ; then
+   sed -i  '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
+fi
+
+exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}

docker-image/v0.12/alpine-file/hooks/post_push

@@ -0,0 +1,15 @@
+#!/bin/bash
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/post_push.erb
+
+set -e
+
+# Parse image name for repo name
+tagStart=$(expr index "$IMAGE_NAME" :)
+repoName=${IMAGE_NAME:0:tagStart-1}
+
+# Tag and push image for each additional tag
+for tag in {v0.12.33-syslog,v0.12-syslog,stable-syslog,syslog}; do
+  docker tag $IMAGE_NAME ${repoName}:${tag}
+  docker push ${repoName}:${tag}
+done

docker-image/v0.12/alpine-file/plugins/parser_kubernetes.rb

@@ -0,0 +1,65 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# The following Fluentd parser plugin, aims to simplify the parsing of multiline
+# logs found in Kubernetes nodes. Since many log files shared the same format and
+# in order to simplify the configuration, this plugin provides a 'kubernetes' format
+# parser (built on top of MultilineParser).
+#
+# When tailing files, this 'kubernetes' format should be applied to the following
+# log file sources:
+#
+#  - /var/log/kubelet.log
+#  - /var/log/kube-proxy.log
+#  - /var/log/kube-apiserver.log
+#  - /var/log/kube-controller-manager.log
+#  - /var/log/kube-scheduler.log
+#  - /var/log/rescheduler.log
+#  - /var/log/glbc.log
+#  - /var/log/cluster-autoscaler.log
+#
+# Usage:
+#
+# ---- fluentd.conf ----
+#
+# <source>
+#    type tail
+#    format kubernetes
+#    path ./kubelet.log
+#    read_from_head yes
+#    tag kubelet
+# </source>
+#
+# ----   EOF       ---
+
+require 'fluent/parser'
+
+module Fluent
+  class KubernetesParser < Fluent::TextParser::MultilineParser
+    Fluent::Plugin.register_parser("kubernetes", self)
+
+    CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
+    CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/}
+    CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"
+
+    def configure(conf)
+      conf['format_firstline'] = CONF_FORMAT_FIRSTLINE
+      conf['format1'] = CONF_FORMAT1
+      conf['time_format'] = CONF_TIME_FORMAT
+      super
+    end
+  end
+end

fluentd-daemonset-file.yaml

@@ -0,0 +1,48 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: fluentd
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-logging
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-logging
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      containers:
+      - name: fluentd
+        image: benhall/fluentd-kubernetes-daemonset:file
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: logstorage
+          mountPath: /var/log/fluent
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: logstorage
+        hostPath:
+          path: /tmp/data/