feat(charts): Set new migration values and use bootloader image (#8843)

charts/airbyte/README.md

@@ -29,7 +29,7 @@
 | `webapp.replicaCount`        | Number of webapp replicas                                        | `1`              |
 | `webapp.image.repository`    | The repository to use for the airbyte webapp image.              | `airbyte/webapp` |
 | `webapp.image.pullPolicy`    | the pull policy to use for the airbyte webapp image              | `IfNotPresent`   |
-| `webapp.image.tag`           | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`  |
+| `webapp.image.tag`           | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`   |
 | `webapp.podAnnotations`      | Add extra annotations to the webapp pod(s)                       | `{}`             |
 | `webapp.service.type`        | The service type to use for the webapp service                   | `ClusterIP`      |
 | `webapp.service.port`        | The service port to expose the webapp on                         | `80`             |
@@ -55,7 +55,7 @@
 | `scheduler.replicaCount`       | Number of scheduler replicas                                        | `1`                 |
 | `scheduler.image.repository`   | The repository to use for the airbyte scheduler image.              | `airbyte/scheduler` |
 | `scheduler.image.pullPolicy`   | the pull policy to use for the airbyte scheduler image              | `IfNotPresent`      |
-| `scheduler.image.tag`          | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`     |
+| `scheduler.image.tag`          | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`      |
 | `scheduler.podAnnotations`     | Add extra annotations to the scheduler pod                          | `{}`                |
 | `scheduler.resources.limits`   | The resources limits for the scheduler container                    | `{}`                |
 | `scheduler.resources.requests` | The requested resources for the scheduler container                 | `{}`                |
@@ -86,7 +86,7 @@
 | `server.replicaCount`                       | Number of server replicas                                        | `1`              |
 | `server.image.repository`                   | The repository to use for the airbyte server image.              | `airbyte/server` |
 | `server.image.pullPolicy`                   | the pull policy to use for the airbyte server image              | `IfNotPresent`   |
-| `server.image.tag`                          | The airbyte server image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`  |
+| `server.image.tag`                          | The airbyte server image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`   |
 | `server.podAnnotations`                     | Add extra annotations to the server pod                          | `{}`             |
 | `server.livenessProbe.enabled`              | Enable livenessProbe on the server                               | `true`           |
 | `server.livenessProbe.initialDelaySeconds`  | Initial delay seconds for livenessProbe                          | `30`             |
@@ -120,7 +120,7 @@
 | `worker.replicaCount`                       | Number of worker replicas                                        | `1`              |
 | `worker.image.repository`                   | The repository to use for the airbyte worker image.              | `airbyte/worker` |
 | `worker.image.pullPolicy`                   | the pull policy to use for the airbyte worker image              | `IfNotPresent`   |
-| `worker.image.tag`                          | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`  |
+| `worker.image.tag`                          | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.34.4-alpha`   |
 | `worker.podAnnotations`                     | Add extra annotations to the worker pod(s)                       | `{}`             |
 | `worker.livenessProbe.enabled`              | Enable livenessProbe on the worker                               | `true`           |
 | `worker.livenessProbe.initialDelaySeconds`  | Initial delay seconds for livenessProbe                          | `30`             |
@@ -142,6 +142,15 @@
 | `worker.extraEnv`                           | Additional env vars for worker pod(s).                           | `[]`             |
 
 
+### Bootloader Parameters
+
+| Name                          | Description                                                          | Value                |
+| ----------------------------- | -------------------------------------------------------------------- | -------------------- |
+| `bootloader.image.repository` | The repository to use for the airbyte bootloader image.              | `airbyte/bootloader` |
+| `bootloader.image.pullPolicy` | the pull policy to use for the airbyte bootloader image              | `IfNotPresent`       |
+| `bootloader.image.tag`        | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.34.0-alpha`       |
+
+
 ### Temporal parameters
 
 | Name                        | Description                                   | Value                   |
@@ -188,5 +197,15 @@
 | `logs.s3.enabled`            | Switch to enable or disable custom S3 Log location     | `false`            |
 | `logs.s3.bucket`             | Bucket name where logs should be stored                | `airbyte-dev-logs` |
 | `logs.s3.bucketRegion`       | Region of the bucket (must be empty if using minio)    | `""`               |
+| `logs.gcs.bucket`            | GCS bucket name                                        | `""`               |
+| `logs.gcs.credentials`       | The path the GCS creds are written to                  | `""`               |
+
+
+### Minio chart overwrites
+
+| Name                       | Description      | Value      |
+| -------------------------- | ---------------- | ---------- |
+| `minio.accessKey.password` | Minio Access Key | `minio`    |
+| `minio.secretKey.password` | Minio Secret Key | `minio123` |
 
 

charts/airbyte/templates/_helpers.tpl

@@ -207,6 +207,13 @@ Returns the Airbyte worker Image
 {{- include "common.images.image" (dict "imageRoot" .Values.worker.image "global" .Values.global) -}}
 {{- end -}}
 
+{{/*
+Returns the Airbyte Bootloader Image
+*/}}
+{{- define "airbyte.bootloaderImage" -}}
+{{- include "common.images.image" (dict "imageRoot" .Values.bootloader.image "global" .Values.global) -}}
+{{- end -}}
+
 {{/*
 Returns the Temporal Image. TODO: This will probably be replaced if we move to using temporal as a dependency, like minio and postgres.
 */}}

charts/airbyte/templates/bootloader/pod.yaml

@@ -0,0 +1,54 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ include "common.names.fullname" . }}-bootloader
+  labels:
+    {{- include "airbyte.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+spec:
+  restartPolicy: Never
+  containers:
+    - name: airbyte-bootloader-container
+      image: {{ include "airbyte.bootloaderImage" . }}
+      imagePullPolicy: "{{ .Values.bootloader.image.pullPolicy }}"
+      env:
+        - name: AIRBYTE_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: AIRBYTE_VERSION
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: DATABASE_PORT
+        {{- if .Values.postgresql.enabled }}
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: DATABASE_PASSWORD
+        {{- else }}
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "airbyte.postgresql.secretName" . }}
+              key: {{ include "airbyte.database.existingsecret.key" . }}
+        {{- end }}
+        - name: DATABASE_URL
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: DATABASE_URL
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: DATABASE_USER

charts/airbyte/templates/env-configmap.yaml

@@ -8,6 +8,7 @@ data:
   AWS_ACCESS_KEY_ID: {{ .Values.logs.accessKey.password }}
   AWS_SECRET_ACCESS_KEY: {{ .Values.logs.secretKey.password }}
   CONFIG_ROOT: /configs
+  CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.30.22.001"
   DATA_DOCKER_MOUNT: airbyte_data
   DATABASE_DB: {{ include "airbyte.database.name" . }}
   DATABASE_HOST: {{ include "airbyte.database.host" . }}
@@ -25,6 +26,7 @@ data:
   JOB_POD_MAIN_CONTAINER_CPU_REQUEST: ""
   JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT: ""
   JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST: ""
+  JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.29.15.001"
   LOCAL_ROOT: /tmp/airbyte_local
   RUN_DATABASE_MIGRATION_ON_STARTUP: "true"
   S3_LOG_BUCKET: {{ .Values.logs.s3.bucket }}

charts/airbyte/templates/scheduler/deployment.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:

charts/airbyte/templates/server/deployment.yaml

@@ -1,4 +1,3 @@
-
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -153,6 +152,16 @@ spec:
             configMapKeyRef:
               name: airbyte-env
               key: GCS_LOG_BUCKET
+        - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
+        - name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: airbyte-env
+              key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
         {{- if .Values.server.extraEnv }}
         {{ .Values.server.extraEnv | toYaml | nindent 8 }}
         {{- end }}

charts/airbyte/templates/tests/test-webapp.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Pod
 metadata:

charts/airbyte/templates/webapp/ingress.yaml

@@ -1,3 +1,4 @@
+---
 {{- if .Values.webapp.ingress.enabled -}}
 {{- $fullName := include "airbyte.fullname" . -}}
 {{- $svcPort := .Values.webapp.service.port -}}

charts/airbyte/values.yaml

@@ -437,6 +437,17 @@ worker:
   ##   value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule"
   extraEnv: []
 
+## @section Bootloader Parameters
+
+bootloader:
+  ## @param bootloader.image.repository The repository to use for the airbyte bootloader image.
+  ## @param bootloader.image.pullPolicy the pull policy to use for the airbyte bootloader image
+  ## @param bootloader.image.tag The airbyte bootloader image tag. Defaults to the chart's AppVersion
+  image:
+    repository: airbyte/bootloader
+    pullPolicy: IfNotPresent
+    tag: 0.34.0-alpha
+
 ## @section Temporal parameters
 ## TODO: Move to consuming temporal from a dedicated helm chart
 
@@ -542,14 +553,17 @@ logs:
     enabled: false
     bucket: airbyte-dev-logs
     bucketRegion: ""
- 
+
   ## Google Cloud Storage (GCS) Log Location Configuration
-  ## @param gcs.bucket GCS bucket name
-  ## @param gcs.credentials The path the GCS creds are written to
+  ## @param logs.gcs.bucket GCS bucket name
+  ## @param logs.gcs.credentials The path the GCS creds are written to
   gcs:
     bucket: ""
     credentials: ""
 
+## @section Minio chart overwrites
+## @param minio.accessKey.password Minio Access Key
+## @param minio.secretKey.password Minio Secret Key
 minio:
   accessKey:
     password: minio