From 4bccf9b8ac654fdd7bae9c816965bcded970cf45 Mon Sep 17 00:00:00 2001 From: Luke Amdor Date: Thu, 22 Mar 2018 16:34:10 -0500 Subject: [PATCH 1/7] cp configmap elements over to config volume as configmap vol is RO --- kafka/10broker-config.yml | 1 + kafka/50kafka.yml | 8 ++++++-- zookeeper/10zookeeper-config.yml | 1 + zookeeper/50pzoo.yml | 8 ++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index debfda14..661eb520 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -9,6 +9,7 @@ data: set -x KAFKA_BROKER_ID=${HOSTNAME##*-} + cp -Lur /etc/kafka-configmap/* /etc/kafka/ sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties LABELS="kafka-broker-id=$KAFKA_BROKER_ID" diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index e8b3abf4..dfa00713 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -34,8 +34,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - command: ['/bin/bash', '/etc/kafka/init.sh'] + command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: + - name: configmap + mountPath: /etc/kafka-configmap - name: config mountPath: /etc/kafka containers: @@ -70,9 +72,11 @@ spec: - name: data mountPath: /var/lib/kafka/data volumes: - - name: config + - name: configmap configMap: name: broker-config + - name: config + emptyDir: {} volumeClaimTemplates: - metadata: name: data diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index e796b4ba..06943914 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -11,6 +11,7 @@ data: [ -z "$ID_OFFSET" ] && ID_OFFSET=1 export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid + cp -Lur /etc/kafka-configmap/* /etc/kafka/ sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties zookeeper.properties: |- diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 9d301218..1308739d 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -23,8 +23,10 @@ spec: initContainers: - name: init-config image: solsson/kafka:1.0.1@sha256:1a4689d49d6274ac59b9b740f51b0408e1c90a9b66d16ad114ee9f7193bab111 - command: ['/bin/bash', '/etc/kafka/init.sh'] + command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: + - name: configmap + mountPath: /etc/kafka-configmap - name: config mountPath: /etc/kafka - name: data @@ -61,9 +63,11 @@ spec: - name: data mountPath: /var/lib/zookeeper/data volumes: - - name: config + - name: configmap configMap: name: zookeeper-config + - name: config + emptyDir: {} volumeClaimTemplates: - metadata: name: data From df1b9adee85e56a1d270f450a3f859121df88f44 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Mar 2018 22:00:26 +0100 Subject: [PATCH 2/7] Fixes init for ephemeral zoo too --- zookeeper/51zoo.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 452d923d..6c822ab9 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -23,11 +23,13 @@ spec: initContainers: - name: init-config image: solsson/kafka:1.0.1@sha256:1a4689d49d6274ac59b9b740f51b0408e1c90a9b66d16ad114ee9f7193bab111 - command: ['/bin/bash', '/etc/kafka/init.sh'] + command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] env: - name: ID_OFFSET value: "4" volumeMounts: + - name: configmap + mountPath: /etc/kafka-configmap - name: config mountPath: /etc/kafka - name: data @@ -64,8 +66,10 @@ spec: - name: data mountPath: /var/lib/zookeeper/data volumes: - - name: config + - name: configmap configMap: name: zookeeper-config + - name: config + emptyDir: {} - name: data emptyDir: {} From 2621880f7e56b50ff9d6b16daed2973e182974f2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Mar 2018 21:38:51 +0100 Subject: [PATCH 3/7] Consistently use initutils, with kubectl 1.9.6 --- kafka/50kafka.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index dfa00713..0f74ac7e 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -20,7 +20,7 @@ spec: terminationGracePeriodSeconds: 30 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:c98d7fb5e9365eab391a5dcd4230fc6e72caf929c60f29ff091e3b0215124713 + image: solsson/kafka-initutils@sha256:d9dcee0210b58dd53c8dd328a3929e97eb33ef0c624e95e2ce832024df888a5d env: - name: NODE_NAME valueFrom: diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 1308739d..c087c34a 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:1.0.1@sha256:1a4689d49d6274ac59b9b740f51b0408e1c90a9b66d16ad114ee9f7193bab111 + image: solsson/kafka-initutils@sha256:d9dcee0210b58dd53c8dd328a3929e97eb33ef0c624e95e2ce832024df888a5d command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: - name: configmap diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 6c822ab9..2a2349fe 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:1.0.1@sha256:1a4689d49d6274ac59b9b740f51b0408e1c90a9b66d16ad114ee9f7193bab111 + image: solsson/kafka-initutils@sha256:d9dcee0210b58dd53c8dd328a3929e97eb33ef0c624e95e2ce832024df888a5d command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] env: - name: ID_OFFSET From 827512a3cefdfc31a999440b52fafdc25f1c021b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Mar 2018 22:33:12 +0100 Subject: [PATCH 4/7] Pardon, this is the image with kubectl 1.9.6 --- kafka/50kafka.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 0f74ac7e..c79e6521 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -20,7 +20,7 @@ spec: terminationGracePeriodSeconds: 30 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:d9dcee0210b58dd53c8dd328a3929e97eb33ef0c624e95e2ce832024df888a5d + image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 env: - name: NODE_NAME valueFrom: diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index c087c34a..2859f3d9 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:d9dcee0210b58dd53c8dd328a3929e97eb33ef0c624e95e2ce832024df888a5d + image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: - name: configmap diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 2a2349fe..d0132143 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:d9dcee0210b58dd53c8dd328a3929e97eb33ef0c624e95e2ce832024df888a5d + image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] env: - name: ID_OFFSET From f18e3fca8a5735842575a3bd6f0102fd3fd885c1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Mar 2018 23:03:35 +0100 Subject: [PATCH 5/7] Documents the need for running master until 4.0 is tagged --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bfeefede..7a160afe 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e | tag | k8s ≥ | highlights | | ----- | ------ | ---------- | +| master | 1.9.4 | Required for read-only ConfigMaps, k8s 1.9.4+ | | v3.1 | 1.8 | The painstaking path to `min.insync.replicas`=2 | | v3.0 | 1.8 | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) | | v2.1 | 1.5 | Kafka 1.0, the init script concept | From 10f312b145f0a8379e948585939cbeadf1c09394 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 23 Mar 2018 23:04:44 +0100 Subject: [PATCH 6/7] Spotted a typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a160afe..2cfca85a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ To quote [@arthurk](https://github.com/Yolean/kubernetes-kafka/issues/82#issueco > thanks for creating and maintaining this Kubernetes files, they're up-to-date (unlike the kubernetes contrib files, don't require helm and work great! -## Gettings started +## Getting started We suggest you `apply -f` manifests in the following order: * You choice of storage classes from [./configure](./configure/) From 6c15dabaf74060a3158e2ccaa4e753ccc5ca0bb1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 24 Mar 2018 21:21:00 +0100 Subject: [PATCH 7/7] Heads up to anyone upgrading almost any cluster --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cfca85a..4527db64 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e | tag | k8s ≥ | highlights | | ----- | ------ | ---------- | -| master | 1.9.4 | Required for read-only ConfigMaps, k8s 1.9.4+ | +| master | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) | | v3.1 | 1.8 | The painstaking path to `min.insync.replicas`=2 | | v3.0 | 1.8 | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) | | v2.1 | 1.5 | Kafka 1.0, the init script concept |