Add storage config.

README.md

@@ -20,3 +20,11 @@ To install the quickwit chart:
 To uninstall the chart:
 
     helm delete my-quickwit
+
+## Upgrade helm chart from 0.4.0 to 0.5.0
+
+The way storage config is defined changed and you have to update your helm values to upgrade to 0.5.0.
+
+The changes are:
+- the `config.s3` and `config.azure_blob` values are no more supported. You now have to use the storage config as defined in the [docs](https://quickwit.io/docs/configuration/storage-config).
+- the keys of secrets have changed: `s3.secret_key` is replaced by `storage.s3.secret_access_key` and `azure_blob.access_key` is replace by `storage.azure.access_key`.

charts/quickwit/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: quickwit
 description: Sub-second search & analytics engine on cloud storage.
 type: application
-version: 0.4.0
-appVersion: "v0.6.1"
+version: 0.5.0
+appVersion: "v0.6.4"
 keywords:
   - quickwit
   - search

charts/quickwit/templates/_helpers.tpl

@@ -140,35 +140,27 @@ Quickwit environment
   value: node.yaml
 - name: QW_CLUSTER_ID
   value: {{ .Release.Namespace }}-{{ include "quickwit.fullname" . }}
-{{- with .Values.config.s3 }}
-{{- if .endpoint }}
-- name: QW_S3_ENDPOINT
-  value: {{ .endpoint }}
-{{- end }}
-{{- if .region }}
-- name: AWS_REGION
-  value: {{ .region }}
-{{- end }}
-{{- if and .secret_key .access_key }}
+{{- if ((.Values.config.storage).s3).access_key_id }}
 - name: AWS_ACCESS_KEY_ID
-  value: {{ .access_key }}
+  value: {{ .Values.config.storage.s3.access_key_id }}
+{{- end }}
+{{- if ((.Values.config.storage).s3).secret_access_key }}
 - name: AWS_SECRET_ACCESS_KEY
   valueFrom:
     secretKeyRef:
       name: {{ include "quickwit.fullname" $ }}
-      key: s3.secret_key
-{{- end }}
+      key: storage.s3.secret_access_key
 {{- end }}
-{{- if .Values.config.azure_blob.account_name }}
+{{- if ((.Values.config.storage).azure).account }}
 - name: QW_AZURE_STORAGE_ACCOUNT
-  value: {{ .Values.config.azure_blob.account_name }}
+  value: {{ .Values.config.storage.azure.account }}
 {{- end }}
-{{- if .Values.config.azure_blob.access_key }}
+{{- if ((.Values.config.storage).azure).access_key }}
 - name: QW_AZURE_STORAGE_ACCESS_KEY
   valueFrom:
     secretKeyRef:
       name: {{ include "quickwit.fullname" $ }}
-      key: azure_blob.access_key
+      key: storage.azure.access_key
 {{- end }}
 - name: QW_NODE_ID
   value: "$(POD_NAME)"

charts/quickwit/templates/configmap.yaml

@@ -6,7 +6,7 @@ metadata:
     {{- include "quickwit.labels" . | nindent 4 }}
 data:
   node.yaml: |-
-    version: 0.5
+    version: 0.6
     listen_address: 0.0.0.0
     gossip_listen_port: 7282
     data_dir: /quickwit/qwdata
@@ -15,6 +15,21 @@ data:
     indexer:
       {{- toYaml . | nindent 6 }}
     {{- end }}
+    {{- with .Values.config.storage }}
+    {{- $new_storage := . | deepCopy }}
+    {{- /* s3 access_key_id and secret_access_key are already set by env variables */}}
+    {{ if .s3 }}
+    {{- $_ := unset $new_storage.s3 "access_key_id" }}
+    {{- $_ := unset $new_storage.s3 "secret_access_key" }}
+    {{- end }}
+    {{- /* azure account and access_key are already set by env variables */}}
+    {{ if .azure }}
+    {{- $_ := unset $new_storage.azure "account" }}
+    {{- $_ := unset $new_storage.azure "access_key" }}
+    {{- end }}
+    storage:
+      {{- toYaml $new_storage | nindent 6 }}
+    {{- end }}
     {{- with .Values.config.ingest_api }}
     ingest_api:
       {{- toYaml . | nindent 6 }}

charts/quickwit/templates/secret.yaml

@@ -9,9 +9,9 @@ data:
 {{- if .Values.config.postgres }}
   postgres.password: {{ required "A valid config.postgres.password is required!" .Values.config.postgres.password | b64enc | quote }}
 {{- end }}
-{{- if .Values.config.s3.secret_key }}
-  s3.secret_key: {{ .Values.config.s3.secret_key | b64enc | quote }}
+{{- if ((.Values.config.storage).s3).secret_access_key }}
+  storage.s3.secret_access_key: {{ .Values.config.storage.s3.secret_access_key | b64enc | quote }}
 {{- end }}
-{{- if .Values.config.azure_blob.access_key }}
-  azure_blob.access_key: {{ .Values.config.azure_blob.access_key | b64enc | quote }}
+{{- if ((.Values.config.storage).azure).access_key }}
+  storage.azure.access_key: {{ .Values.config.storage.azure.access_key | b64enc | quote }}
 {{- end }}

charts/quickwit/values.yaml

@@ -6,7 +6,7 @@ image:
   repository: quickwit/quickwit
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  # tag: v0.6.1
+  # tag: v0.6.4
 
 imagePullSecrets: []
 nameOverride: ""
@@ -306,22 +306,27 @@ bootstrap:
 # Quickwit configuration
 config:
   # Metastore configuration.
-  postgres: {}
+  # Could be set either with QW_METASTORE_URI or with the following config:
+  # - If on S3:
+  # metastore_uri: s3://quickwit-indexes
+  # - If on postgres-like db:
+  # postgres: {}
   #   host: ""
   #   port: 5432
   #   database: metastore
   #   username: quickwit
   #   password: ""
 
-  s3: {}
-    # endpoint: "https://s3.eu-west-1.amazonaws.com"
-    # region: eu-west-1
-    # access_key: "my-access-key"
-    # secret_key: "my-secret-key"
-
-  azure_blob: {}
-    # account_name: "my-azure-blob-accout"
-    # access_key: "my-azure-blob-access-key"
+  storage: {}
+    # s3:
+    #   flavor: gcs
+    #   region: eu-west-1
+    #   access_key_id: "my-access-key"
+    #   secret_access_key: "my-secret-key"
+    #   endpoint: https://storage.googleapis.com
+    # azure:
+    #   account: your-azure-account-name
+    #   access_key: your-azure-access-key
 
   default_index_root_uri: s3://quickwit/indexes
   # Indexer settings
@@ -364,7 +369,7 @@ config:
   sources: []
     # - index: my-index
     #   source:
-    #     version: 0.5
+    #     version: 0.6
     #     source_id: my-source
     #     source_type: kafka
     #     num_pipelines: 1