Airbyte to 0.3.7

charts/airbyte/Chart.lock

@@ -1,12 +1,12 @@
 dependencies:
 - name: common
   repository: https://charts.bitnami.com/bitnami
-  version: 1.10.1
+  version: 1.16.0
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
-  version: 10.13.9
+  version: 10.16.2
 - name: minio
   repository: https://charts.bitnami.com/bitnami
-  version: 7.3.2
-digest: sha256:51711092d4fb978667aaeadac5080241db1eb32d4a0d4d25c2a775110d013813
-generated: "2021-11-30T11:54:53.849182827+01:00"
+  version: 11.7.4
+digest: sha256:3b186d0e008cc2c3286827f730542b992320fe6fd0abe52fe60ee9b98b2e7823
+generated: "2022-06-15T11:35:14.642872-04:00"

charts/airbyte/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.1
+version: 0.3.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.35.12-alpha"
+appVersion: "0.39.23-alpha"
 
 dependencies:
   - name: common
@@ -33,7 +33,7 @@ dependencies:
     name: postgresql
     version: 10.x.x
     repository: https://charts.bitnami.com/bitnami
-  - condition: minio.enabled
+  - condition: logs.minio.enabled
     name: minio
-    version: 7.x.x
+    version: 11.x.x
     repository: https://charts.bitnami.com/bitnami

charts/airbyte/files/sweep-pod.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
 get_worker_pods () {
-    kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod --field-selector status.phase!=Running get pods -o go-template --template '{{range .items}} {{.metadata.name}} {{.status.phase}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}'
+    kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \
+      --field-selector status.phase!=Running get pods \
+      -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime}{"\n"}{end}'
 }
 
 delete_worker_pod() {

charts/airbyte/index.yaml

@@ -1,9 +1,81 @@
 apiVersion: v1
 entries:
   airbyte:
+  - apiVersion: v2
+    appVersion: 0.39.23-alpha
+    created: "2022-06-23T13:31:27.013694-04:00"
+    dependencies:
+    - name: common
+      repository: https://charts.bitnami.com/bitnami
+      tags:
+      - bitnami-common
+      version: 1.x.x
+    - condition: postgresql.enabled
+      name: postgresql
+      repository: https://charts.bitnami.com/bitnami
+      version: 10.x.x
+    - condition: logs.minio.enabled
+      name: minio
+      repository: https://charts.bitnami.com/bitnami
+      version: 11.x.x
+    description: Helm chart to deploy airbyte
+    digest: eb2303e0830f0afe3b6081da242dea73e78b25bc69c8835e8beb895a142f452a
+    name: airbyte
+    type: application
+    urls:
+    - airbyte-0.3.7.tgz
+    version: 0.3.7
+  - apiVersion: v2
+    appVersion: 0.39.18-alpha
+    created: "2022-06-23T13:31:26.985638-04:00"
+    dependencies:
+    - name: common
+      repository: https://charts.bitnami.com/bitnami
+      tags:
+      - bitnami-common
+      version: 1.x.x
+    - condition: postgresql.enabled
+      name: postgresql
+      repository: https://charts.bitnami.com/bitnami
+      version: 10.x.x
+    - condition: logs.minio.enabled
+      name: minio
+      repository: https://charts.bitnami.com/bitnami
+      version: 11.x.x
+    description: Helm chart to deploy airbyte
+    digest: 0399945e94c80e55f8eb372ec685676d8e05d22c987b8d1838106cc19a51b1a0
+    name: airbyte
+    type: application
+    urls:
+    - airbyte-0.3.6.tgz
+    version: 0.3.6
+  - apiVersion: v2
+    appVersion: 0.39.18-alpha
+    created: "2022-06-23T13:31:26.964901-04:00"
+    dependencies:
+    - name: common
+      repository: https://charts.bitnami.com/bitnami
+      tags:
+      - bitnami-common
+      version: 1.x.x
+    - condition: postgresql.enabled
+      name: postgresql
+      repository: https://charts.bitnami.com/bitnami
+      version: 10.x.x
+    - condition: logs.minio.enabled
+      name: minio
+      repository: https://charts.bitnami.com/bitnami
+      version: 11.x.x
+    description: Helm chart to deploy airbyte
+    digest: 144075e759cbe852b56d423f47dd796823280db6498c2b795e902359f135bc17
+    name: airbyte
+    type: application
+    urls:
+    - airbyte-0.3.5.tgz
+    version: 0.3.5
   - apiVersion: v2
     appVersion: 0.35.12-alpha
-    created: "2022-01-31T12:14:06.423369-05:00"
+    created: "2022-06-23T13:31:26.950665-04:00"
     dependencies:
     - name: common
       repository: https://charts.bitnami.com/bitnami
@@ -27,7 +99,7 @@ entries:
     version: 0.3.1
   - apiVersion: v2
     appVersion: 0.35.12-alpha
-    created: "2022-01-31T12:14:06.414595-05:00"
+    created: "2022-06-23T13:31:26.936148-04:00"
     dependencies:
     - name: common
       repository: https://charts.bitnami.com/bitnami
@@ -53,11 +125,10 @@ entries:
   - annotations:
       category: Infrastructure
     apiVersion: v2
-    appVersion: 1.10.0
-    created: "2022-01-31T12:14:06.423957-05:00"
-    description: A Library Helm Chart for grouping common logic between bitnami charts.
-      This chart is not deployable by itself.
-    digest: d13f6834adeb4d8d9cf94d2149680b08006543ab88944192f236decebd6313ae
+    appVersion: 1.16.0
+    created: "2022-06-23T13:31:27.01597-04:00"
+    description: A Library Helm Chart for grouping common logic between bitnami charts. This chart is not deployable by itself.
+    digest: 612ba0ee47eecb204499b7021508c8ebff38d8b50899be717de3e44e7aa78aec
     home: https://github.com/bitnami/charts/tree/master/bitnami/common
     icon: https://bitnami.com/downloads/logos/bitnami-mark.png
     keywords:
@@ -67,32 +138,30 @@ entries:
     - function
     - bitnami
     maintainers:
-    - email: [email protected]
-      name: Bitnami
+    - name: Bitnami
+      url: https://github.com/bitnami/charts
     name: common
     sources:
     - https://github.com/bitnami/charts
-    - http://www.bitnami.com/
+    - https://www.bitnami.com/
     type: library
     urls:
-    - charts/common-1.10.1.tgz
-    version: 1.10.1
+    - charts/common-1.16.0.tgz
+    version: 1.16.0
   minio:
   - annotations:
       category: Infrastructure
     apiVersion: v2
-    appVersion: 2021.9.3
-    created: "2022-01-31T12:14:06.426501-05:00"
+    appVersion: 2022.6.11
+    created: "2022-06-23T13:31:27.02132-04:00"
     dependencies:
     - name: common
       repository: https://charts.bitnami.com/bitnami
       tags:
       - bitnami-common
       version: 1.x.x
-    description: Bitnami Object Storage based on MinIO® is an object storage server,
-      compatible with Amazon S3 cloud storage service, mainly used for storing unstructured
-      data (such as photos, videos, log files, etc.)
-    digest: 8eb8c9056b8c2c64570976ad7f51fa7b71486012c8f7ba965100b369fa58ff4b
+    description: MinIO(R) is an object storage server, compatible with Amazon S3 cloud storage service, mainly used for storing unstructured data (such as photos, videos, log files, etc.).
+    digest: 4a4370b34b3380aaa96663429d3c6bd970838855589e4310e6bce968ee9cc1e7
     home: https://github.com/bitnami/charts/tree/master/bitnami/minio
     icon: https://bitnami.com/assets/stacks/minio/img/minio-stack-220x234.png
     keywords:
@@ -102,28 +171,27 @@ entries:
     - s3
     - cluster
     maintainers:
-    - email: [email protected]
-      name: Bitnami
+    - name: Bitnami
+      url: https://github.com/bitnami/charts
     name: minio
     sources:
     - https://github.com/bitnami/bitnami-docker-minio
     - https://min.io
     urls:
-    - charts/minio-7.3.2.tgz
-    version: 7.3.2
+    - charts/minio-11.7.4.tgz
+    version: 11.7.4
   postgresql:
   - annotations:
       category: Database
     apiVersion: v2
     appVersion: 11.14.0
-    created: "2022-01-31T12:14:06.429256-05:00"
+    created: "2022-06-23T13:31:27.025281-04:00"
     dependencies:
     - name: common
       repository: https://charts.bitnami.com/bitnami
       version: 1.x.x
-    description: Chart for PostgreSQL, an object-relational database management system
-      (ORDBMS) with an emphasis on extensibility and on standards-compliance.
-    digest: bf390d423e45c512228ad22c2e5899acfc420c0bb424a169ff8770ce3cb05495
+    description: Chart for PostgreSQL, an object-relational database management system (ORDBMS) with an emphasis on extensibility and on standards-compliance.
+    digest: 6012be8e28cad57f87320c1b035b668c6d6d0325e384ce4732ec154e452dbca0
     home: https://github.com/bitnami/charts/tree/master/bitnami/postgresql
     icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png
     keywords:
@@ -143,6 +211,6 @@ entries:
     - https://github.com/bitnami/bitnami-docker-postgresql
     - https://www.postgresql.org/
     urls:
-    - charts/postgresql-10.13.9.tgz
-    version: 10.13.9
-generated: "2022-01-31T12:14:06.403751-05:00"
+    - charts/postgresql-10.16.2.tgz
+    version: 10.16.2
+generated: "2022-06-23T13:31:26.920795-04:00"

charts/airbyte/templates/_helpers.tpl

@@ -70,25 +70,21 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
-
 {{/*
-Get the Postgresql credentials secret.
+Get the Postgresql credentials secret name.
 */}}
-{{- define "airbyte.postgresql.secretName" -}}
-{{- if and (.Values.postgresql.enabled) (not .Values.postgresql.existingSecret) -}}
-    {{- printf "%s" (include "airbyte.postgresql.fullname" .) -}}
-{{- else if and (.Values.postgresql.enabled) (.Values.postgresql.existingSecret) -}}
-    {{- printf "%s" .Values.postgresql.existingSecret -}}
+{{- define "airbyte.database.secret.name" -}}
+{{- if .Values.postgresql.enabled -}}
+    {{ template "postgresql.secretName" .Subcharts.postgresql }}
 {{- else }}
     {{- if .Values.externalDatabase.existingSecret -}}
         {{- printf "%s" .Values.externalDatabase.existingSecret -}}
     {{- else -}}
-        {{ printf "%s-%s" .Release.Name "externaldb" }}
+        {{ printf "%s-%s" (include "common.names.fullname" .) "secrets" }}
     {{- end -}}
 {{- end -}}
 {{- end -}}
 
-
 {{/*
 Add environment variables to configure database values
 */}}
@@ -111,9 +107,9 @@ Add environment variables to configure database values
 {{- end -}}
 
 {{/*
-Add environment variables to configure database values
+Get the Postgresql credentials secret password key
 */}}
-{{- define "airbyte.database.existingsecret.key" -}}
+{{- define "airbyte.database.secret.passwordKey" -}}
 {{- if .Values.postgresql.enabled -}}
     {{- printf "%s" "postgresql-password" -}}
 {{- else -}}
@@ -124,7 +120,7 @@ Add environment variables to configure database values
             {{- printf "%s" "postgresql-password" -}}
         {{- end -}}
     {{- else -}}
-        {{- printf "%s" "postgresql-password" -}}
+        {{- printf "%s" "DATABASE_PASSWORD" -}}
     {{- end -}}
 {{- end -}}
 {{- end -}}
@@ -162,7 +158,7 @@ Add environment variables to configure minio
 {{- if .Values.logs.minio.enabled -}}
     {{- printf "http://%s:%d" (include "airbyte.minio.fullname" .) 9000 -}}
 {{- else if .Values.logs.externalMinio.enabled -}}
-    {{- printf "http://%s:%d" .Values.logs.externalMinio.host .Values.logs.externalMinio.port -}}
+    {{- printf "http://%s:%g" .Values.logs.externalMinio.host .Values.logs.externalMinio.port -}}
 {{- else -}}
     {{- printf "" -}}
 {{- end -}}
@@ -172,6 +168,17 @@ Add environment variables to configure minio
 {{- ternary "true" "" (or .Values.logs.minio.enabled .Values.logs.externalMinio.enabled) -}}
 {{- end -}}
 
+{{/*
+Returns the GCP credentials path
+*/}}
+{{- define "airbyte.gcpLogCredentialsPath" -}}
+{{- if .Values.logs.gcs.credentialsJson }}
+    {{- printf "%s" "/secrets/gcs-log-creds/gcp.json" -}}
+{{- else -}}
+    {{- printf "%s" .Values.logs.gcs.credentials -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Returns the Airbyte Scheduler Image
 */}}
@@ -220,3 +227,25 @@ Returns the Temporal Image. TODO: This will probably be replaced if we move to u
 {{- define "airbyte.temporalImage" -}}
 {{- include "common.images.image" (dict "imageRoot" .Values.temporal.image "global" .Values.global) -}}
 {{- end -}}
+
+{{/*
+Construct comma separated list of key/value pairs from object (useful for ENV var values)
+*/}}
+{{- define "airbyte.flattenMap" -}}
+{{- $kvList := list -}}
+{{- range $key, $value := . -}}
+{{- $kvList = printf "%s=%s" $key $value | mustAppend $kvList -}}
+{{- end -}}
+{{ join "," $kvList }}
+{{- end -}}
+
+{{/*
+Construct semi-colon delimited list of comma separated key/value pairs from array of objects (useful for ENV var values)
+*/}}
+{{- define "airbyte.flattenArrayMap" -}}
+{{- $mapList := list -}}
+{{- range $element := . -}}
+{{- $mapList = include "airbyte.flattenMap" $element | mustAppend $mapList -}}
+{{- end -}}
+{{ join ";" $mapList }}
+{{- end -}}