From b09c605af964c43ed89ef0885593c4c23976bd26 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 10:24:09 +0100
Subject: [PATCH 01/26] Changes folder name for merge to 1.8 master

---
 {addon-logs => logs-streaming}/logs-kube-kafka.yml         | 0
 {addon-logs => logs-streaming}/topic-ops-kube-logs-raw.yml | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename {addon-logs => logs-streaming}/logs-kube-kafka.yml (100%)
 rename {addon-logs => logs-streaming}/topic-ops-kube-logs-raw.yml (100%)

diff --git a/addon-logs/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
similarity index 100%
rename from addon-logs/logs-kube-kafka.yml
rename to logs-streaming/logs-kube-kafka.yml
diff --git a/addon-logs/topic-ops-kube-logs-raw.yml b/logs-streaming/topic-ops-kube-logs-raw.yml
similarity index 100%
rename from addon-logs/topic-ops-kube-logs-raw.yml
rename to logs-streaming/topic-ops-kube-logs-raw.yml

From d274f48ec7201cf8a55d60914bf345b48127f7a5 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 10:29:28 +0100
Subject: [PATCH 02/26] No reason to run in kube-system namespace

---
 logs-streaming/logs-kube-kafka.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
index 7b1f49ee..2aa2a5b4 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka.yml
@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
   name: logs-kafka
-  namespace: kube-system
+  namespace: kafka
 spec:
   template:
     metadata:

From 975ac91bb5afe3b60c620e572c4a7f99c830813b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 10:37:02 +0100
Subject: [PATCH 03/26] Manifest updated to k8s 1.8

---
 logs-streaming/logs-kube-kafka.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
index 2aa2a5b4..68826809 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka.yml
@@ -1,9 +1,14 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
   name: logs-kafka
   namespace: kafka
 spec:
+  selector:
+    matchLabels:
+      k8s-app: logs-kafka
+      version: v1
+      kubernetes.io/cluster-service: "true"
   template:
     metadata:
       labels:

From 6a7ca4496231e585a03cab49ced78ac0e3545112 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 10:52:00 +0100
Subject: [PATCH 04/26] Same level of producer info as in events-kube, and ...

hopefully still no log collection recursion.
---
 logs-streaming/logs-kube-kafka.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
index 68826809..664ad333 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka.yml
@@ -33,7 +33,9 @@ spec:
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
           -t ops-kube-logs-raw-001
           -P
-          #-z snappy
+          -z snappy
+          -v
+          -d broker,topic
         resources:
           limits:
             memory: 100Mi

From ebfbfc0f1a2029e010c149b04c7531e31e7079c8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 10:52:57 +0100
Subject: [PATCH 05/26] Fixes log reading in GKE's 1.8.1 nodes

---
 logs-streaming/logs-kube-kafka.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
index 664ad333..f9d0e8a5 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka.yml
@@ -46,8 +46,14 @@ spec:
         - name: varlog
           mountPath: /var/log
           readOnly: true
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
       terminationGracePeriodSeconds: 10
       volumes:
       - name: varlog
         hostPath:
           path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers

From a6c522fdff0838473d44efc211d621ef01e2efae Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Oct 2017 17:42:37 +0200
Subject: [PATCH 06/26] test stub

---
 addon-logs/test/logs-aggregated.yml | 67 +++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 addon-logs/test/logs-aggregated.yml

diff --git a/addon-logs/test/logs-aggregated.yml b/addon-logs/test/logs-aggregated.yml
new file mode 100644
index 00000000..99345702
--- /dev/null
+++ b/addon-logs/test/logs-aggregated.yml
@@ -0,0 +1,67 @@
+---
+kind: ConfigMap
+metadata:
+  name: test-logs-aggregated
+  namespace: test-kafka
+apiVersion: v1
+data:
+
+  setup.sh: |-
+    touch /tmp/testlog
+
+    tail -f /tmp/testlog
+
+  continue.sh: |-
+    exit 0
+
+  run.sh: |-
+    exit 0
+
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: test-logs-aggregated
+  namespace: test-kafka
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        test-target: kafka
+        test-type: readiness
+    spec:
+      containers:
+      - name: testcase
+        image: solsson/kafkacat@sha256:decb4e4a0e722c8e202bd05e81c43b94d87c6cba977f90d16623e74027045000
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+        - name: ZOOKEEPER
+          value: zookeeper.kafka.svc.cluster.local:2181
+        # Test set up
+        command:
+        - /bin/bash
+        - -e
+        - /test/setup.sh
+        # Test run, again and again
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/run.sh
+        # Test quit on nonzero exit
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/continue.sh
+        volumeMounts:
+        - name: config
+          mountPath: /test
+      volumes:
+      - name: config
+        configMap:
+          name: test-logs-aggregated

From a693a22008f1ccbad5e383ef6ce27604f433b1fc Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Oct 2017 17:43:40 +0200
Subject: [PATCH 07/26] Prints only some logs to test log, to avoid an infinite
 loop of aggregation

---
 addon-logs/test/logs-aggregated.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/addon-logs/test/logs-aggregated.yml b/addon-logs/test/logs-aggregated.yml
index 99345702..170a2f9d 100644
--- a/addon-logs/test/logs-aggregated.yml
+++ b/addon-logs/test/logs-aggregated.yml
@@ -12,6 +12,10 @@ data:
     tail -f /tmp/testlog
 
   continue.sh: |-
+
+    echo "--- $(date -u --iso-8601='ns') ---" >> /tmp/testlog
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '%t [%p] %o: %k = %s\n' -o -10 -e >> /tmp/testlog
+
     exit 0
 
   run.sh: |-
@@ -33,12 +37,12 @@ spec:
     spec:
       containers:
       - name: testcase
-        image: solsson/kafkacat@sha256:decb4e4a0e722c8e202bd05e81c43b94d87c6cba977f90d16623e74027045000
+        image: solsson/kafkacat@sha256:9cee76d02eee590c4874c42ca2e10de22d15c2dfa5b67205e35089a49cfba1f4
         env:
         - name: BOOTSTRAP
           value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
-        - name: ZOOKEEPER
-          value: zookeeper.kafka.svc.cluster.local:2181
+        - name: TOPIC
+          value: ops-kube-logs-raw-001
         # Test set up
         command:
         - /bin/bash

From efa3676b922693709a9df5bd3e34541ac045a75d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Oct 2017 07:41:27 +0200
Subject: [PATCH 08/26] Makes test printout compatible with jq

---
 addon-logs/test/logs-aggregated.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/addon-logs/test/logs-aggregated.yml b/addon-logs/test/logs-aggregated.yml
index 170a2f9d..31edd075 100644
--- a/addon-logs/test/logs-aggregated.yml
+++ b/addon-logs/test/logs-aggregated.yml
@@ -13,8 +13,8 @@ data:
 
   continue.sh: |-
 
-    echo "--- $(date -u --iso-8601='ns') ---" >> /tmp/testlog
-    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '%t [%p] %o: %k = %s\n' -o -10 -e >> /tmp/testlog
+    echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" >> /tmp/testlog
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e >> /tmp/testlog
 
     exit 0
 

From 0b0cbfd3f2af3001a57aa1dfe732c79e80e08ec8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Oct 2017 17:08:05 +0200
Subject: [PATCH 09/26] Intuitively the exit condition should be last, I think

---
 test/basic-produce-consume.yml | 6 +++---
 test/basic-with-kafkacat.yml   | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/basic-produce-consume.yml b/test/basic-produce-consume.yml
index 128a00a9..66dda328 100644
--- a/test/basic-produce-consume.yml
+++ b/test/basic-produce-consume.yml
@@ -18,9 +18,6 @@ data:
 
     tail -f /tmp/testlog
 
-  continue.sh: |-
-    exit 0
-
   run.sh: |-
     exec >> /tmp/testlog
     exec 2>&1
@@ -37,6 +34,9 @@ data:
 
     exit 0
 
+  continue.sh: |-
+    exit 0
+
 ---
 apiVersion: apps/v1beta1
 kind: Deployment
diff --git a/test/basic-with-kafkacat.yml b/test/basic-with-kafkacat.yml
index 405a7980..cb6f6675 100644
--- a/test/basic-with-kafkacat.yml
+++ b/test/basic-with-kafkacat.yml
@@ -10,9 +10,6 @@ data:
     touch /tmp/testlog
     tail -f /tmp/testlog
 
-  continue.sh: |-
-    exit 0
-
   run.sh: |-
     exec >> /tmp/testlog
     exec 2>&1
@@ -24,6 +21,9 @@ data:
 
     exit 0
 
+  continue.sh: |-
+    exit 0
+
 ---
 apiVersion: batch/v1
 kind: Job

From 0806ce0593bee0eba5b27415e3d1af1f721b46ec Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 11:00:36 +0100
Subject: [PATCH 10/26] Uses latest kafkacat build for logs test

---
 addon-logs/test/logs-aggregated.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/addon-logs/test/logs-aggregated.yml b/addon-logs/test/logs-aggregated.yml
index 31edd075..e4d18075 100644
--- a/addon-logs/test/logs-aggregated.yml
+++ b/addon-logs/test/logs-aggregated.yml
@@ -37,7 +37,7 @@ spec:
     spec:
       containers:
       - name: testcase
-        image: solsson/kafkacat@sha256:9cee76d02eee590c4874c42ca2e10de22d15c2dfa5b67205e35089a49cfba1f4
+        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
         env:
         - name: BOOTSTRAP
           value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092

From 959d4c5cafa70db1aaef1f8ea4d1d7008dc449bb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 11:00:57 +0100
Subject: [PATCH 11/26] Refreshes test syntax from Yolean/kube-test

---
 addon-logs/test/logs-aggregated.yml | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/addon-logs/test/logs-aggregated.yml b/addon-logs/test/logs-aggregated.yml
index e4d18075..dc6ed42c 100644
--- a/addon-logs/test/logs-aggregated.yml
+++ b/addon-logs/test/logs-aggregated.yml
@@ -11,14 +11,16 @@ data:
 
     tail -f /tmp/testlog
 
-  continue.sh: |-
+  test.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
 
-    echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" >> /tmp/testlog
-    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e >> /tmp/testlog
+    echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}"
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e
 
     exit 0
 
-  run.sh: |-
+  quit-on-nonzero-exit.sh: |-
     exit 0
 
 ---
@@ -43,25 +45,22 @@ spec:
           value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
         - name: TOPIC
           value: ops-kube-logs-raw-001
-        # Test set up
         command:
         - /bin/bash
         - -e
         - /test/setup.sh
-        # Test run, again and again
         readinessProbe:
           exec:
             command:
             - /bin/bash
             - -e
-            - /test/run.sh
-        # Test quit on nonzero exit
+            - /test/test.sh
         livenessProbe:
           exec:
             command:
             - /bin/bash
             - -e
-            - /test/continue.sh
+            - /test/quit-on-nonzero-exit.sh
         volumeMounts:
         - name: config
           mountPath: /test

From eb2f8958af34ead5533f68fed783f749154b56fb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 11:08:23 +0100
Subject: [PATCH 12/26] Updates manifest to k8s 1.8

---
 {addon-logs => logs-streaming}/test/logs-aggregated.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
 rename {addon-logs => logs-streaming}/test/logs-aggregated.yml (91%)

diff --git a/addon-logs/test/logs-aggregated.yml b/logs-streaming/test/logs-aggregated.yml
similarity index 91%
rename from addon-logs/test/logs-aggregated.yml
rename to logs-streaming/test/logs-aggregated.yml
index dc6ed42c..6356b686 100644
--- a/addon-logs/test/logs-aggregated.yml
+++ b/logs-streaming/test/logs-aggregated.yml
@@ -24,17 +24,21 @@ data:
     exit 0
 
 ---
-apiVersion: apps/v1beta1
+apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
   name: test-logs-aggregated
   namespace: test-kafka
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      test-target: logs-streaming
+      test-type: readiness
   template:
     metadata:
       labels:
-        test-target: kafka
+        test-target: logs-streaming
         test-type: readiness
     spec:
       containers:

From 55e0e20addfd4f203eb15f3852f8b8572b499f8d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 11:11:45 +0100
Subject: [PATCH 13/26] As in #60: Need >1 replicas for producer to handle pod
 rescheduling

---
 logs-streaming/topic-ops-kube-logs-raw.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/logs-streaming/topic-ops-kube-logs-raw.yml b/logs-streaming/topic-ops-kube-logs-raw.yml
index 9fd9418b..f89f5e58 100644
--- a/logs-streaming/topic-ops-kube-logs-raw.yml
+++ b/logs-streaming/topic-ops-kube-logs-raw.yml
@@ -25,8 +25,7 @@ spec:
         - --partitions
         - "1"
         - --replication-factor
-        - "1"
+        - "2"
         - --config
-        # this might be eight days
-        - retention.ms=69125000
+        - retention.ms=6912500000
       restartPolicy: Never

From 224cfbe1af2abdc896d0ed598e16065eb42ba30d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 11:17:29 +0100
Subject: [PATCH 14/26] Can't output as JSON unless aggregation produces JSON

---
 logs-streaming/test/logs-aggregated.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logs-streaming/test/logs-aggregated.yml b/logs-streaming/test/logs-aggregated.yml
index 6356b686..6807d226 100644
--- a/logs-streaming/test/logs-aggregated.yml
+++ b/logs-streaming/test/logs-aggregated.yml
@@ -16,7 +16,7 @@ data:
     exec 2>&1
 
     echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}"
-    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '%t@%p [%o] %k: %s\n' -o -10 -e
 
     exit 0
 

From 9446d4a9d32af760e1e87b27afa4c0be60fe154f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 11:17:49 +0100
Subject: [PATCH 15/26] Increases readability of test output

---
 logs-streaming/test/logs-aggregated.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/logs-streaming/test/logs-aggregated.yml b/logs-streaming/test/logs-aggregated.yml
index 6807d226..4da336cf 100644
--- a/logs-streaming/test/logs-aggregated.yml
+++ b/logs-streaming/test/logs-aggregated.yml
@@ -15,8 +15,9 @@ data:
     exec >> /tmp/testlog
     exec 2>&1
 
+    echo ""
     echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}"
-    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '%t@%p [%o] %k: %s\n' -o -10 -e
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '\n%t@%p [%o] %k: %s\n' -o -10 -e
 
     exit 0
 

From 0dcde3e7af05f166890228c351f232c1b82a858d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 15:14:39 +0100
Subject: [PATCH 16/26] We'll lose some lines at restart, but we'll pick up new
 containers ...

actually getting amazingly close to the result from filebeat or fluentd.

The `==> [filename] <==` messages give the filename for coming nessages
up to the next such line (thanks to `tail`) and the filenames all have the pattern
pod-name_namespace_container-name_id.log

A streams application could process this and pass messages to another
topic, after adding labels and annotations from kube API, either as very long keys
or wrapping as json like fluentd or filebeat does.
---
 logs-streaming/logs-kube-kafka.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
index f9d0e8a5..423c2d71 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka.yml
@@ -23,6 +23,7 @@ spec:
         - sh
         - -ec
         - >
+          find /var/log/containers/ -maxdepth 1 -name *.log | tee /tmp/start-list;
           cd /var/log/containers/;
           tail
           -n 0
@@ -36,6 +37,14 @@ spec:
           -z snappy
           -v
           -d broker,topic
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - find /var/log/containers/ -maxdepth 1 -name *.log | diff - /tmp/start-list
+          initialDelaySeconds: 5
+          periodSeconds: 5
         resources:
           limits:
             memory: 100Mi

From 9166ebd9872c82772bd631d248db0179c777367e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 15:28:48 +0100
Subject: [PATCH 17/26] Got categorized as CrashLoopBackoff

---
 logs-streaming/logs-kube-kafka.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka.yml
index 423c2d71..56fbeea1 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka.yml
@@ -43,7 +43,7 @@ spec:
             - /bin/sh
             - -c
             - find /var/log/containers/ -maxdepth 1 -name *.log | diff - /tmp/start-list
-          initialDelaySeconds: 5
+          initialDelaySeconds: 15
           periodSeconds: 5
         resources:
           limits:

From 5d3fd7c9b97fc7ab3eba4daf71169a1359590a75 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 15:49:09 +0100
Subject: [PATCH 18/26] Leaves room for log collectors, now that I like this
 one ...

although it will never be reliable in clusters with a lot of scheduling going on
---
 .../{logs-kube-kafka.yml => logs-kube-kafka-raw.yml}   |  9 +++++++--
 .../test/{logs-aggregated.yml => logs-raw.yml}         | 10 +++++-----
 2 files changed, 12 insertions(+), 7 deletions(-)
 rename logs-streaming/{logs-kube-kafka.yml => logs-kube-kafka-raw.yml} (89%)
 rename logs-streaming/test/{logs-aggregated.yml => logs-raw.yml} (90%)

diff --git a/logs-streaming/logs-kube-kafka.yml b/logs-streaming/logs-kube-kafka-raw.yml
similarity index 89%
rename from logs-streaming/logs-kube-kafka.yml
rename to logs-streaming/logs-kube-kafka-raw.yml
index 56fbeea1..de56a8be 100644
--- a/logs-streaming/logs-kube-kafka.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -1,18 +1,22 @@
 apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
-  name: logs-kafka
+  name: logs-kafka-raw
   namespace: kafka
 spec:
   selector:
     matchLabels:
       k8s-app: logs-kafka
+      log-wrapping: none
+      container-delimiters: tail
       version: v1
       kubernetes.io/cluster-service: "true"
   template:
     metadata:
       labels:
         k8s-app: logs-kafka
+        log-wrapping: none
+        container-delimiters: tail
         version: v1
         kubernetes.io/cluster-service: "true"
     spec:
@@ -43,7 +47,8 @@ spec:
             - /bin/sh
             - -c
             - find /var/log/containers/ -maxdepth 1 -name *.log | diff - /tmp/start-list
-          initialDelaySeconds: 15
+          failureThreshold: 1
+          initialDelaySeconds: 25
           periodSeconds: 5
         resources:
           limits:
diff --git a/logs-streaming/test/logs-aggregated.yml b/logs-streaming/test/logs-raw.yml
similarity index 90%
rename from logs-streaming/test/logs-aggregated.yml
rename to logs-streaming/test/logs-raw.yml
index 4da336cf..dbe9dc1d 100644
--- a/logs-streaming/test/logs-aggregated.yml
+++ b/logs-streaming/test/logs-raw.yml
@@ -1,7 +1,7 @@
 ---
 kind: ConfigMap
 metadata:
-  name: test-logs-aggregated
+  name: logs-raw
   namespace: test-kafka
 apiVersion: v1
 data:
@@ -28,18 +28,18 @@ data:
 apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
-  name: test-logs-aggregated
+  name: logs-raw
   namespace: test-kafka
 spec:
   replicas: 1
   selector:
     matchLabels:
-      test-target: logs-streaming
+      test-target: logs-streaming-raw
       test-type: readiness
   template:
     metadata:
       labels:
-        test-target: logs-streaming
+        test-target: logs-streaming-raw
         test-type: readiness
     spec:
       containers:
@@ -72,4 +72,4 @@ spec:
       volumes:
       - name: config
         configMap:
-          name: test-logs-aggregated
+          name: logs-raw

From 986d10b9c0f4e6c673d71ed9eb5e0a7884eab05d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 15:51:27 +0100
Subject: [PATCH 19/26] Rock solid :) ...

taking into account that pod's own log file might not be there at start.

Now I actually get 1 logs-kafka-raw per 1 container deletion.
---
 logs-streaming/logs-kube-kafka-raw.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index de56a8be..28cf3e39 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -27,7 +27,7 @@ spec:
         - sh
         - -ec
         - >
-          find /var/log/containers/ -maxdepth 1 -name *.log | tee /tmp/start-list;
+          find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | tee /tmp/start-list;
           cd /var/log/containers/;
           tail
           -n 0
@@ -46,9 +46,9 @@ spec:
             command:
             - /bin/sh
             - -c
-            - find /var/log/containers/ -maxdepth 1 -name *.log | diff - /tmp/start-list
+            - find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | diff - /tmp/start-list
           failureThreshold: 1
-          initialDelaySeconds: 25
+          initialDelaySeconds: 5
           periodSeconds: 5
         resources:
           limits:

From 8528f0b5125f10783db49c9af9268e77f42055c1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 15:57:52 +0100
Subject: [PATCH 20/26] Beat this, fluent-bit :) ...

(I know you're close)
---
 logs-streaming/logs-kube-kafka-raw.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index 28cf3e39..d8bc8eb1 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -51,11 +51,12 @@ spec:
           initialDelaySeconds: 5
           periodSeconds: 5
         resources:
-          limits:
-            memory: 100Mi
           requests:
-            cpu: 100m
-            memory: 100Mi
+            cpu: 2m
+            memory: 4Mi
+          limits:
+            cpu: 10m
+            memory: 4Mi
         volumeMounts:
         - name: varlog
           mountPath: /var/log

From 68b829b7ac667943b4b4a75527107363723e6a57 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 6 Nov 2017 20:40:20 +0100
Subject: [PATCH 21/26] Moves "raw" to the logs-kafka namespace

---
 logs-streaming/logs-kube-kafka-raw.yml     | 2 +-
 logs-streaming/topic-ops-kube-logs-raw.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index d8bc8eb1..7dd09f12 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -2,7 +2,7 @@ apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
   name: logs-kafka-raw
-  namespace: kafka
+  namespace: logs-kafka
 spec:
   selector:
     matchLabels:
diff --git a/logs-streaming/topic-ops-kube-logs-raw.yml b/logs-streaming/topic-ops-kube-logs-raw.yml
index f89f5e58..a271cbac 100644
--- a/logs-streaming/topic-ops-kube-logs-raw.yml
+++ b/logs-streaming/topic-ops-kube-logs-raw.yml
@@ -2,7 +2,7 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: topic-ops-kube-logs-raw
-  namespace: kafka
+  namespace: logs-kafka
 spec:
   template:
     metadata:
@@ -17,7 +17,7 @@ spec:
         command:
         - ./bin/kafka-topics.sh
         - --zookeeper
-        - zookeeper:2181
+        - zookeeper.kafka:2181
         - --create
         - --if-not-exists
         - --topic

From 97f446d9d1cdbe36eb208ba188da9b8b358dd5f5 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 6 Nov 2017 20:41:07 +0100
Subject: [PATCH 22/26] Had problems with pods not terminating

and though it looked like unhealthy nodes I see no reason
for a longer grace period with neither tail nor kafkacat.
---
 logs-streaming/logs-kube-kafka-raw.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index 7dd09f12..b697f40b 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -20,6 +20,8 @@ spec:
         version: v1
         kubernetes.io/cluster-service: "true"
     spec:
+      # maybe the /bin/sh does not forward signals?
+      terminationGracePeriodSeconds: 5
       containers:
       - name: kafkacat
         image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef

From de0b1f5ba3f375ad5bbc3e32c6344c01e1537c36 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 6 Nov 2017 20:42:06 +0100
Subject: [PATCH 23/26] Uses node name as key so we can preserver ordering

Addresses https://github.com/Yolean/kubernetes-kafka/pull/88#issuecomment-341957997.

I deemed it safe to assume that operational daemonset pods never co-exist on a node.

Tests https://github.com/edenhill/kafkacat/pull/123,
as does 53f355a.
---
 logs-streaming/logs-kube-kafka-raw.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index b697f40b..05928e92 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -24,7 +24,12 @@ spec:
       terminationGracePeriodSeconds: 5
       containers:
       - name: kafkacat
-        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        image: solsson/kafkacat-fixedkey@sha256:c9c78aef3090519b64ecc279c06df6914c94720a27469200c079f1871b10fa8f
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         command:
         - sh
         - -ec
@@ -43,6 +48,7 @@ spec:
           -z snappy
           -v
           -d broker,topic
+          -F $NODE_NAME
         livenessProbe:
           exec:
             command:

From cc99542e5aac5e61dcbb13923de1bcab2f36bcac Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 6 Nov 2017 21:24:38 +0100
Subject: [PATCH 24/26] Latest https://github.com/edenhill/kafkacat/pull/123

---
 logs-streaming/logs-kube-kafka-raw.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index 05928e92..440ac83b 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -24,7 +24,7 @@ spec:
       terminationGracePeriodSeconds: 5
       containers:
       - name: kafkacat
-        image: solsson/kafkacat-fixedkey@sha256:c9c78aef3090519b64ecc279c06df6914c94720a27469200c079f1871b10fa8f
+        image: solsson/kafkacat-fixedkey@sha256:e23753bc0900e7b95d29f8c9cb05440eb27d854f367d407ce4ab2955093ad360
         env:
         - name: NODE_NAME
           valueFrom:
@@ -48,7 +48,7 @@ spec:
           -z snappy
           -v
           -d broker,topic
-          -F $NODE_NAME
+          -k $NODE_NAME
         livenessProbe:
           exec:
             command:

From 64ab61f7079e2415571a6ced7f29b7abb404056a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 7 Nov 2017 09:41:08 +0100
Subject: [PATCH 25/26] kafkacat based on same image as solsson/kafka:1.0.0

---
 logs-streaming/logs-kube-kafka-raw.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index 440ac83b..7362866d 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -24,7 +24,7 @@ spec:
       terminationGracePeriodSeconds: 5
       containers:
       - name: kafkacat
-        image: solsson/kafkacat-fixedkey@sha256:e23753bc0900e7b95d29f8c9cb05440eb27d854f367d407ce4ab2955093ad360
+        image: solsson/kafkacat-fixedkey@sha256:fb27acf4c7ca25eca6a6cffe145cb492ee4d4c87cba0459787fb43f8040f6d7b
         env:
         - name: NODE_NAME
           valueFrom:

From 17b7ccc75e2479b6a981bffc772a72a879f06b44 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 9 Nov 2017 13:11:23 +0100
Subject: [PATCH 26/26] Kafkacat support for -k has been merged now

---
 logs-streaming/logs-kube-kafka-raw.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml
index 7362866d..4470ed56 100644
--- a/logs-streaming/logs-kube-kafka-raw.yml
+++ b/logs-streaming/logs-kube-kafka-raw.yml
@@ -24,7 +24,7 @@ spec:
       terminationGracePeriodSeconds: 5
       containers:
       - name: kafkacat
-        image: solsson/kafkacat-fixedkey@sha256:fb27acf4c7ca25eca6a6cffe145cb492ee4d4c87cba0459787fb43f8040f6d7b
+        image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9
         env:
         - name: NODE_NAME
           valueFrom: