From 889bd0d47235cb1ab5a7b52439f5b8df61026a03 Mon Sep 17 00:00:00 2001 From: Mikhail Koniakhin Date: Tue, 5 Nov 2024 06:24:25 +0200 Subject: [PATCH] feat: recovery support multi hosts and ports of external kafka cluster (#1588) --- charts/sentry/README.md | 16 +++++++++------- charts/sentry/templates/NOTES.txt | 6 ++++++ charts/sentry/templates/_helper.tpl | 16 ++++++++-------- .../templates/hooks/sentry-db-check.job.yaml | 4 ++-- charts/sentry/values.yaml | 13 +++++++++++-- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/charts/sentry/README.md b/charts/sentry/README.md index 8d5d894eb..65dd66122 100644 --- a/charts/sentry/README.md +++ b/charts/sentry/README.md @@ -68,6 +68,7 @@ Note: this table is incomplete, so have a look at the values.yaml in case you mi | externalClickhouse.singleNode | bool | `true` | | | externalClickhouse.tcpPort | int | `9000` | | | externalClickhouse.username | string | `"default"` | | +| externalKafka.cluster | list | `[]` | Multi hosts and ports of external Kafka | | externalKafka.host | string | `"kafka-confluent"` | Hostname or IP address of external Kafka | | externalKafka.port | int | `9092` | Port for external Kafka | | externalKafka.compression.type | string | `""` | Compression type for Kafka messages ('gzip', 'snappy', 'lz4', 'zstd') | @@ -1249,13 +1250,14 @@ or you can feed in a cluster of Kafka instances like below: ```yaml externalKafka: - ## List of Hostnames or ip addresses of external kafka - - host: "233.5.100.28" - port: 9092 - - host: "233.5.100.29" - port: 9092 - - host: "233.5.100.30" - port: 9092 + cluster: + ## List of Hostnames or ip addresses and ports of external kafka + - host: "233.5.100.28" + port: 9092 + - host: "kafka-confluent-2" + port: 9093 + - host: "kafka-confluent-3" + port: 9094 ``` ## External Postgres configuration diff --git a/charts/sentry/templates/NOTES.txt b/charts/sentry/templates/NOTES.txt index f99c764ad..d9cd60aba 100644 --- a/charts/sentry/templates/NOTES.txt +++ b/charts/sentry/templates/NOTES.txt @@ -1,3 +1,9 @@ * When running upgrades, make sure to give back the `system.secretKey` value. kubectl -n {{ .Release.Namespace }} get configmap {{ template "sentry.fullname" . }}-sentry -o json | grep -m1 -Po '(?<=system.secret-key: )[^\\]*' + +{{ if not (.Values.kafka.enabled) }} +* Sentry use external kafka: + +{{ template "sentry.kafka.bootstrap_servers_string" . }} +{{ end -}} diff --git a/charts/sentry/templates/_helper.tpl b/charts/sentry/templates/_helper.tpl index 8d2366b15..e148a5625 100644 --- a/charts/sentry/templates/_helper.tpl +++ b/charts/sentry/templates/_helper.tpl @@ -418,7 +418,7 @@ Set Kafka Confluent host {{- define "sentry.kafka.host" -}} {{- if .Values.kafka.enabled -}} {{- template "sentry.kafka.fullname" . -}} -{{- else if and (.Values.externalKafka) (not (kindIs "slice" .Values.externalKafka)) -}} +{{- else if and (.Values.externalKafka) (not (.Values.externalKafka.cluster)) -}} {{ required "A valid .Values.externalKafka.host is required" .Values.externalKafka.host }} {{- end -}} {{- end -}} @@ -429,7 +429,7 @@ Set Kafka Confluent port {{- define "sentry.kafka.port" -}} {{- if and (.Values.kafka.enabled) (.Values.kafka.service.ports.client) -}} {{- .Values.kafka.service.ports.client }} -{{- else if and (.Values.externalKafka) (not (kindIs "slice" .Values.externalKafka)) -}} +{{- else if and (.Values.externalKafka) (not (.Values.externalKafka.cluster)) -}} {{ required "A valid .Values.externalKafka.port is required" .Values.externalKafka.port }} {{- end -}} {{- end -}} @@ -440,7 +440,7 @@ Set Kafka Confluent Controller port {{- define "sentry.kafka.controller_port" -}} {{- if and (.Values.kafka.enabled) (.Values.kafka.service.ports.controller ) -}} {{- .Values.kafka.service.ports.controller }} -{{- else if and (.Values.externalKafka) (not (kindIs "slice" .Values.externalKafka)) -}} +{{- else if and (.Values.externalKafka) (not (.Values.externalKafka.cluster)) -}} {{ required "A valid .Values.externalKafka.port is required" .Values.externalKafka.port }} {{- end -}} {{- end -}} @@ -449,10 +449,10 @@ Set Kafka Confluent Controller port Set Kafka bootstrap servers string */}} {{- define "sentry.kafka.bootstrap_servers_string" -}} -{{- if or (.Values.kafka.enabled) (not (kindIs "slice" .Values.externalKafka)) -}} +{{- if or (.Values.kafka.enabled) (not (.Values.externalKafka.cluster)) -}} {{ printf "%s:%s" (include "sentry.kafka.host" .) (include "sentry.kafka.port" .) }} {{- else -}} -{{- range $index, $elem := .Values.externalKafka -}} +{{- range $index, $elem := .Values.externalKafka.cluster -}} {{- if $index -}},{{- end -}}{{ printf "%s:%s" $elem.host (toString $elem.port) }} {{- end -}} {{- end -}} @@ -460,9 +460,9 @@ Set Kafka bootstrap servers string {{/* SASL auth setings for Kafka: -* https://github.com/getsentry/snuba/blob/24.7.1/snuba/settings/__init__.py#L219-L229 -* https://github.com/getsentry/sentry/blob/24.7.1/src/sentry/utils/kafka_config.py#L9-L34 -* https://github.com/getsentry/sentry/blob/24.7.1/src/sentry/conf/server.py#L2827-L2836 +* https://github.com/getsentry/snuba/blob/24.9.0/snuba/settings/__init__.py#L220-L230 +* https://github.com/getsentry/sentry/blob/24.9.0/src/sentry/utils/kafka_config.py#L9-L34 +* https://github.com/getsentry/sentry/blob/24.9.0/src/sentry/conf/server.py#L2844-L2853 */}} {{/* diff --git a/charts/sentry/templates/hooks/sentry-db-check.job.yaml b/charts/sentry/templates/hooks/sentry-db-check.job.yaml index df4f3c286..e8eb690cb 100644 --- a/charts/sentry/templates/hooks/sentry-db-check.job.yaml +++ b/charts/sentry/templates/hooks/sentry-db-check.job.yaml @@ -162,14 +162,14 @@ spec: done echo "Kraft controllers are up" {{- end }} - {{- else if (not (kindIs "slice" .Values.externalKafka)) }} + {{- else if (not (.Values.externalKafka.cluster)) }} KAFKA_HOST={{ .Values.externalKafka.host }} if ! nc -z "$KAFKA_HOST" {{ $kafkaPort }}; then KAFKA_STATUS=0 echo "$KAFKA_HOST is not available yet" fi {{- else }} - {{- range $elem := .Values.externalKafka }} + {{- range $elem := .Values.externalKafka.cluster }} KAFKA_HOST={{ $elem.host }} if ! nc -z "$KAFKA_HOST" {{ $elem.port }}; then KAFKA_STATUS=0 diff --git a/charts/sentry/values.yaml b/charts/sentry/values.yaml index ceb4e2023..364e5e7ff 100644 --- a/charts/sentry/values.yaml +++ b/charts/sentry/values.yaml @@ -2179,10 +2179,19 @@ kafka: ## This value is only used when kafka.enabled is set to false ## externalKafka: - ## Hostname or ip address of external kafka + ## Multi hosts and ports of external kafka ## + # cluster: + # - host: "233.5.100.28" + # port: 9092 + # - host: "kafka-confluent-2" + # port: 9093 + # - host: "kafka-confluent-3" + # port: 9094 + ## Or Hostname (ip address) of external kafka # host: "kafka-confluent" - port: 9092 + ## and port of external kafka + # port: 9092 compression: type: # 'gzip', 'snappy', 'lz4', 'zstd' message: