From a45297dbfc21fc6459dd9af5503806e5356bf062 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:15:42 +0100 Subject: [PATCH 01/10] Adds linkedin/Burrow for use with port-forward 8000 --- linkedin-burrow/10-burrow-config.yml | 36 ++++++++++++++++++++++++++++ linkedin-burrow/50-burrow.yml | 27 +++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 linkedin-burrow/10-burrow-config.yml create mode 100644 linkedin-burrow/50-burrow.yml diff --git a/linkedin-burrow/10-burrow-config.yml b/linkedin-burrow/10-burrow-config.yml new file mode 100644 index 00000000..62ed62ed --- /dev/null +++ b/linkedin-burrow/10-burrow-config.yml @@ -0,0 +1,36 @@ +kind: ConfigMap +metadata: + name: burrow-config + namespace: kafka +apiVersion: v1 +data: + burrow.toml: |- + [zookeeper] + servers=[ "zookeeper:2181" ] + timeout=6 + root-path="/burrow" + + [cluster.local] + class-name="kafka" + servers=[ "kafka:9092" ] + topic-refresh=60 + offset-refresh=30 + + [consumer.local] + class-name="kafka" + cluster="local" + servers=[ "kafka:9092" ] + group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-whitelist="" + + [consumer.local_zk] + class-name="kafka_zk" + cluster="local" + servers=[ "zookeeper:2181" ] + zookeeper-path="/local" + zookeeper-timeout=30 + group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-whitelist="" + + [httpserver.default] + address=":8000" diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml new file mode 100644 index 00000000..c7d88d21 --- /dev/null +++ b/linkedin-burrow/50-burrow.yml @@ -0,0 +1,27 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: burrow + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: burrow + template: + metadata: + labels: + app: burrow + spec: + containers: + - name: kafka-manager + image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef + ports: + - containerPort: 8000 + volumeMounts: + - name: config + mountPath: /etc/burrow + volumes: + - name: config + configMap: + name: burrow-config From 43c277ddb0b2f038a4facd407aadc2553724eb1d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:16:34 +0100 Subject: [PATCH 02/10] Lists actual brokers --- linkedin-burrow/10-burrow-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkedin-burrow/10-burrow-config.yml b/linkedin-burrow/10-burrow-config.yml index 62ed62ed..53db4e69 100644 --- a/linkedin-burrow/10-burrow-config.yml +++ b/linkedin-burrow/10-burrow-config.yml @@ -12,14 +12,14 @@ data: [cluster.local] class-name="kafka" - servers=[ "kafka:9092" ] + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] topic-refresh=60 offset-refresh=30 [consumer.local] class-name="kafka" cluster="local" - servers=[ "kafka:9092" ] + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" group-whitelist="" From f84bd724a1a23d995a34850ae34c887ec7dc34bf Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:17:15 +0100 Subject: [PATCH 03/10] fixes container name --- linkedin-burrow/50-burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml index c7d88d21..57ff62a8 100644 --- a/linkedin-burrow/50-burrow.yml +++ b/linkedin-burrow/50-burrow.yml @@ -14,7 +14,7 @@ spec: app: burrow spec: containers: - - name: kafka-manager + - name: burrow image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef ports: - containerPort: 8000 From 545193d425b809b1cec64670d0366ac0ca301542 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:25:52 +0100 Subject: [PATCH 04/10] Adds prometheus exporter, but sadly incompatible with current Burrow https://github.com/jirwin/burrow_exporter/issues/8 --- linkedin-burrow/50-burrow.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml index 57ff62a8..aa7ce6ce 100644 --- a/linkedin-burrow/50-burrow.yml +++ b/linkedin-burrow/50-burrow.yml @@ -12,6 +12,9 @@ spec: metadata: labels: app: burrow + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" spec: containers: - name: burrow @@ -21,6 +24,17 @@ spec: volumeMounts: - name: config mountPath: /etc/burrow + - name: prom + image: solsson/burrow-exporter@sha256:2358412bc4757b7bd459452a9d759a923fc5ba93b26f5fb5b3d1ed8a0189417c + ports: + - containerPort: 8080 + env: + - name: BURROW_ADDR + value: http://localhost:8000 + - name: METRICS_ADDR + value: 0.0.0.0:8080 + - name: INTERVAL + value: "30" volumes: - name: config configMap: From 8b21daf46aff1b4d54dc25e49584194c9593c228 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:46:38 +0100 Subject: [PATCH 05/10] The build from search-and-replace in github.com/jirwin/burrow_exporter/pull/9 --- linkedin-burrow/50-burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml index aa7ce6ce..7af69227 100644 --- a/linkedin-burrow/50-burrow.yml +++ b/linkedin-burrow/50-burrow.yml @@ -25,7 +25,7 @@ spec: - name: config mountPath: /etc/burrow - name: prom - image: solsson/burrow-exporter@sha256:2358412bc4757b7bd459452a9d759a923fc5ba93b26f5fb5b3d1ed8a0189417c + image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 ports: - containerPort: 8080 env: From b3950835b49d2444b1c390fb0d9033c24b52693a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 06:30:14 +0100 Subject: [PATCH 06/10] Adds a service. Ordering is ok with plain file names here. --- .../{10-burrow-config.yml => burrow-config.yml} | 0 linkedin-burrow/burrow-service.yml | 12 ++++++++++++ linkedin-burrow/{50-burrow.yml => burrow.yml} | 0 3 files changed, 12 insertions(+) rename linkedin-burrow/{10-burrow-config.yml => burrow-config.yml} (100%) create mode 100644 linkedin-burrow/burrow-service.yml rename linkedin-burrow/{50-burrow.yml => burrow.yml} (100%) diff --git a/linkedin-burrow/10-burrow-config.yml b/linkedin-burrow/burrow-config.yml similarity index 100% rename from linkedin-burrow/10-burrow-config.yml rename to linkedin-burrow/burrow-config.yml diff --git a/linkedin-burrow/burrow-service.yml b/linkedin-burrow/burrow-service.yml new file mode 100644 index 00000000..f9fc7b49 --- /dev/null +++ b/linkedin-burrow/burrow-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: burrow + namespace: kafka +spec: + selector: + app: burrow + ports: + - protocol: TCP + port: 80 + targetPort: 8000 diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/burrow.yml similarity index 100% rename from linkedin-burrow/50-burrow.yml rename to linkedin-burrow/burrow.yml From 18322c3d72f6e31042f2d3d0765111a15d04fc28 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 07:38:35 +0100 Subject: [PATCH 07/10] Lets us see all consumers, for now --- linkedin-burrow/burrow-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml index 53db4e69..5fe6dda1 100644 --- a/linkedin-burrow/burrow-config.yml +++ b/linkedin-burrow/burrow-config.yml @@ -20,7 +20,7 @@ data: class-name="kafka" cluster="local" servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] - group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-blacklist="" group-whitelist="" [consumer.local_zk] @@ -29,7 +29,7 @@ data: servers=[ "zookeeper:2181" ] zookeeper-path="/local" zookeeper-timeout=30 - group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-blacklist="" group-whitelist="" [httpserver.default] From 181432c4fbd2281887398067661dda70b23a143a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 11:50:49 +0100 Subject: [PATCH 08/10] Adds a frontend container. No need to scale it indivudually. --- linkedin-burrow/burrow-service.yml | 10 ++++++++-- linkedin-burrow/burrow.yml | 23 +++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/linkedin-burrow/burrow-service.yml b/linkedin-burrow/burrow-service.yml index f9fc7b49..15eac06a 100644 --- a/linkedin-burrow/burrow-service.yml +++ b/linkedin-burrow/burrow-service.yml @@ -7,6 +7,12 @@ spec: selector: app: burrow ports: - - protocol: TCP + - name: web + protocol: TCP port: 80 - targetPort: 8000 + - name: api + protocol: TCP + port: 8000 + - name: prometheus + protocol: TCP + port: 8080 diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index 7af69227..bb43dd19 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -20,14 +20,24 @@ spec: - name: burrow image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef ports: - - containerPort: 8000 + - name: api + containerPort: 8000 + readinessProbe: + httpGet: + path: /burrow/admin + port: 8000 + livenessProbe: + httpGet: + path: /burrow/admin + port: 8000 volumeMounts: - name: config mountPath: /etc/burrow - name: prom image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 ports: - - containerPort: 8080 + - name: prometheus + containerPort: 8080 env: - name: BURROW_ADDR value: http://localhost:8000 @@ -35,6 +45,15 @@ spec: value: 0.0.0.0:8080 - name: INTERVAL value: "30" + - name: dashboard + image: joway/burrow-dashboard@sha256:b4cce87c8264119d73f9a6df5d787ea811ce2138d39ca6cd56525bcfbb5169bf + env: + - name: BURROW_BACKEND + value: http://localhost:8000 + ports: + - name: web + containerPort: 80 + protocol: TCP volumes: - name: config configMap: From 1dbe418d5561be9ae8f4cb2a3edd77fa50b0ea43 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 12:17:58 +0100 Subject: [PATCH 09/10] Applies the fix for v3 API so now we export five metrics https://github.com/jirwin/burrow_exporter/pull/9 --- linkedin-burrow/burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index bb43dd19..bdaebb63 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -34,7 +34,7 @@ spec: - name: config mountPath: /etc/burrow - name: prom - image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 + image: solsson/burrow-exporter:api-v3@sha256:dd690c04ae31c62e4d7d5398f24f368fb2c48046ec3ae68bbc7582c114819a7b ports: - name: prometheus containerPort: 8080 From 29eca40589a083fc06d77dca285ad13eb4f15423 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 31 Jan 2018 16:16:05 +0100 Subject: [PATCH 10/10] Upgrades to current Burrow master, a91cf4d --- linkedin-burrow/burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index bdaebb63..2e8c6b5c 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -18,7 +18,7 @@ spec: spec: containers: - name: burrow - image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef + image: solsson/burrow@sha256:a6fe5d1268a7e7d58f7f960697fc8fda3e6861a2653fe71135a2207c9cf18cf0 ports: - name: api containerPort: 8000