Evaluate different log aggregation/streaming approaches

logs-streaming/logs-kube-kafka-raw.yml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  name: logs-kafka-raw
+  namespace: logs-kafka
+spec:
+  selector:
+    matchLabels:
+      k8s-app: logs-kafka
+      log-wrapping: none
+      container-delimiters: tail
+      version: v1
+      kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      labels:
+        k8s-app: logs-kafka
+        log-wrapping: none
+        container-delimiters: tail
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      # maybe the /bin/sh does not forward signals?
+      terminationGracePeriodSeconds: 5
+      containers:
+      - name: kafkacat
+        image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        command:
+        - sh
+        - -ec
+        - >
+          find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | tee /tmp/start-list;
+          cd /var/log/containers/;
+          tail
+          -n 0
+          --follow=name
+          *.log
+          |
+          kafkacat
+          -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+          -t ops-kube-logs-raw-001
+          -P
+          -z snappy
+          -v
+          -d broker,topic
+          -k $NODE_NAME
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | diff - /tmp/start-list
+          failureThreshold: 1
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        resources:
+          requests:
+            cpu: 2m
+            memory: 4Mi
+          limits:
+            cpu: 10m
+            memory: 4Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers

logs-streaming/test/logs-raw.yml

@@ -0,0 +1,75 @@
+---
+kind: ConfigMap
+metadata:
+  name: logs-raw
+  namespace: test-kafka
+apiVersion: v1
+data:
+
+  setup.sh: |-
+    touch /tmp/testlog
+
+    tail -f /tmp/testlog
+
+  test.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    echo ""
+    echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}"
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '\n%t@%p [%o] %k: %s\n' -o -10 -e
+
+    exit 0
+
+  quit-on-nonzero-exit.sh: |-
+    exit 0
+
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: logs-raw
+  namespace: test-kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      test-target: logs-streaming-raw
+      test-type: readiness
+  template:
+    metadata:
+      labels:
+        test-target: logs-streaming-raw
+        test-type: readiness
+    spec:
+      containers:
+      - name: testcase
+        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+        - name: TOPIC
+          value: ops-kube-logs-raw-001
+        command:
+        - /bin/bash
+        - -e
+        - /test/setup.sh
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/test.sh
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/quit-on-nonzero-exit.sh
+        volumeMounts:
+        - name: config
+          mountPath: /test
+      volumes:
+      - name: config
+        configMap:
+          name: logs-raw

addon-logs/topic-ops-kube-logs-raw.yml → logs-streaming/topic-ops-kube-logs-raw.yml

@@ -2,7 +2,7 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: topic-ops-kube-logs-raw
-  namespace: kafka
+  namespace: logs-kafka
 spec:
   template:
     metadata:
@@ -17,16 +17,15 @@ spec:
         command:
         - ./bin/kafka-topics.sh
         - --zookeeper
-        - zookeeper:2181
+        - zookeeper.kafka:2181
         - --create
         - --if-not-exists
         - --topic
         - ops-kube-logs-raw-001
         - --partitions
         - "1"
         - --replication-factor
-        - "1"
+        - "2"
         - --config
-        # this might be eight days
-        - retention.ms=69125000
+        - retention.ms=6912500000
       restartPolicy: Never

test/basic-produce-consume.yml

@@ -18,9 +18,6 @@ data:
 
     tail -f /tmp/testlog
 
-  continue.sh: |-
-    exit 0
-
   run.sh: |-
     exec >> /tmp/testlog
     exec 2>&1
@@ -37,6 +34,9 @@ data:
 
     exit 0
 
+  continue.sh: |-
+    exit 0
+
 ---
 apiVersion: apps/v1beta1
 kind: Deployment

test/basic-with-kafkacat.yml

@@ -10,9 +10,6 @@ data:
     touch /tmp/testlog
     tail -f /tmp/testlog
 
-  continue.sh: |-
-    exit 0
-
   run.sh: |-
     exec >> /tmp/testlog
     exec 2>&1
@@ -24,6 +21,9 @@ data:
 
     exit 0
 
+  continue.sh: |-
+    exit 0
+
 ---
 apiVersion: batch/v1
 kind: Job