diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index d5ae2c8..4cbf758 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: common repository: https://charts.bitnami.com/bitnami - version: 1.10.1 + version: 1.16.0 - name: postgresql repository: https://charts.bitnami.com/bitnami - version: 10.13.9 + version: 10.16.2 - name: minio repository: https://charts.bitnami.com/bitnami - version: 7.3.2 -digest: sha256:51711092d4fb978667aaeadac5080241db1eb32d4a0d4d25c2a775110d013813 -generated: "2021-11-30T11:54:53.849182827+01:00" + version: 11.7.4 +digest: sha256:3b186d0e008cc2c3286827f730542b992320fe6fd0abe52fe60ee9b98b2e7823 +generated: "2022-06-15T11:35:14.642872-04:00" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 0dde693..8d722d8 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.1 +version: 0.3.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.35.12-alpha" +appVersion: "0.39.23-alpha" dependencies: - name: common @@ -33,7 +33,7 @@ dependencies: name: postgresql version: 10.x.x repository: https://charts.bitnami.com/bitnami - - condition: minio.enabled + - condition: logs.minio.enabled name: minio - version: 7.x.x + version: 11.x.x repository: https://charts.bitnami.com/bitnami diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 4528501..5790cf4 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,13 +1,14 @@ # airbyte +Helm charts for Airbyte. + ## Parameters ### Global Parameters -| Name | Description | Value | -| ---------------------- | -------------------------------------------- | ----- | -| `global.imageRegistry` | Global Docker image registry | `""` | -| `global.storageClass` | Global StorageClass for Persistent Volume(s) | `""` | +| Name | Description | Value | +| ---------------------- | ---------------------------- | ----- | +| `global.imageRegistry` | Global Docker image registry | `""` | ### Common Parameters @@ -24,59 +25,75 @@ ### Webapp Parameters -| Name | Description | Value | -| ---------------------------- | ---------------------------------------------------------------- | ---------------- | -| `webapp.replicaCount` | Number of webapp replicas | `1` | -| `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | -| `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | -| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.35.12-alpha` | -| `webapp.podAnnotations` | Add extra annotations to the webapp pod(s) | `{}` | -| `webapp.service.type` | The service type to use for the webapp service | `ClusterIP` | -| `webapp.service.port` | The service port to expose the webapp on | `80` | -| `webapp.resources.limits` | The resources limits for the Web container | `{}` | -| `webapp.resources.requests` | The requested resources for the Web container | `{}` | -| `webapp.nodeSelector` | Node labels for pod assignment | `{}` | -| `webapp.tolerations` | Tolerations for webapp pod assignment. | `[]` | -| `webapp.ingress.enabled` | Set to true to enable ingress record generation | `false` | -| `webapp.ingress.className` | Specifies ingressClassName for clusters >= 1.18+ | `""` | -| `webapp.ingress.annotations` | Ingress annotations done as key:value pairs | `{}` | -| `webapp.ingress.hosts` | The list of hostnames to be covered with this ingress record. | `[]` | -| `webapp.ingress.tls` | Custom ingress TLS configuration | `[]` | -| `webapp.api.url` | The webapp API url. | `/api/v1/` | -| `webapp.isDemo` | Set to true if this is a demo | `false` | -| `webapp.fullstory.enabled` | Whether or not to enable fullstory | `false` | -| `webapp.extraEnv` | Additional env vars for webapp pod(s). | `[]` | - - -### Scheduler Parameters - -| Name | Description | Value | -| ------------------------------ | ------------------------------------------------------------------- | ------------------- | -| `scheduler.replicaCount` | Number of scheduler replicas | `1` | -| `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | -| `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | -| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.35.12-alpha` | -| `scheduler.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | -| `scheduler.resources.limits` | The resources limits for the scheduler container | `{}` | -| `scheduler.resources.requests` | The requested resources for the scheduler container | `{}` | -| `scheduler.nodeSelector` | Node labels for pod assignment | `{}` | -| `scheduler.tolerations` | Tolerations for scheduler pod assignment. | `[]` | -| `scheduler.log.level` | The log level to log at. | `INFO` | -| `scheduler.extraEnv` | Additional env vars for scheduler pod(s). | `[]` | +| Name | Description | Value | +| ------------------------------------------- | ---------------------------------------------------------------- | ---------------- | +| `webapp.replicaCount` | Number of webapp replicas | `1` | +| `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | +| `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | +| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | +| `webapp.podAnnotations` | Add extra annotations to the webapp pod(s) | `{}` | +| `webapp.containerSecurityContext` | Security context for the container | `{}` | +| `webapp.livenessProbe.enabled` | Enable livenessProbe on the webapp | `true` | +| `webapp.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | +| `webapp.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `webapp.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `webapp.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `3` | +| `webapp.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `webapp.readinessProbe.enabled` | Enable readinessProbe on the webapp | `true` | +| `webapp.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `10` | +| `webapp.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `webapp.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `webapp.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `webapp.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `webapp.service.type` | The service type to use for the webapp service | `ClusterIP` | +| `webapp.service.port` | The service port to expose the webapp on | `80` | +| `webapp.service.annotations` | Annotations for the webapp service resource | `{}` | +| `webapp.resources.limits` | The resources limits for the Web container | `{}` | +| `webapp.resources.requests` | The requested resources for the Web container | `{}` | +| `webapp.nodeSelector` | Node labels for pod assignment | `{}` | +| `webapp.tolerations` | Tolerations for webapp pod assignment. | `[]` | +| `webapp.affinity` | Affinity and anti-affinity for webapp pod assignment. | `{}` | +| `webapp.ingress.enabled` | Set to true to enable ingress record generation | `false` | +| `webapp.ingress.className` | Specifies ingressClassName for clusters >= 1.18+ | `""` | +| `webapp.ingress.annotations` | Ingress annotations done as key:value pairs | `{}` | +| `webapp.ingress.hosts` | The list of hostnames to be covered with this ingress record. | `[]` | +| `webapp.ingress.tls` | Custom ingress TLS configuration | `[]` | +| `webapp.api.url` | The webapp API url. | `/api/v1/` | +| `webapp.isDemo` | Set to true if this is a demo | `false` | +| `webapp.fullstory.enabled` | Whether or not to enable fullstory | `false` | +| `webapp.extraEnv` | Additional env vars for webapp pod(s). | `[]` | +| `webapp.extraVolumeMounts` | Additional volumeMounts for webapp container(s). | `[]` | +| `webapp.extraVolumes` | Additional volumes for webapp pod(s). | `[]` | ### Pod Sweeper parameters -| Name | Description | Value | -| ------------------------------- | ---------------------------------------------------- | ----------------- | -| `podSweeper.image.repository` | The image repository to use for the pod sweeper | `bitnami/kubectl` | -| `podSweeper.image.pullPolicy` | The pull policy for the pod sweeper image | `IfNotPresent` | -| `podSweeper.image.tag` | The pod sweeper image tag to use | `latest` | -| `podSweeper.podAnnotations` | Add extra annotations to the podSweeper pod | `{}` | -| `podSweeper.resources.limits` | The resources limits for the podSweeper container | `{}` | -| `podSweeper.resources.requests` | The requested resources for the podSweeper container | `{}` | -| `podSweeper.nodeSelector` | Node labels for pod assignment | `{}` | -| `podSweeper.tolerations` | Tolerations for podSweeper pod assignment. | `[]` | +| Name | Description | Value | +| ----------------------------------------------- | --------------------------------------------------------- | ----------------- | +| `podSweeper.image.repository` | The image repository to use for the pod sweeper | `bitnami/kubectl` | +| `podSweeper.image.pullPolicy` | The pull policy for the pod sweeper image | `IfNotPresent` | +| `podSweeper.image.tag` | The pod sweeper image tag to use | `latest` | +| `podSweeper.podAnnotations` | Add extra annotations to the podSweeper pod | `{}` | +| `podSweeper.containerSecurityContext` | Security context for the container | `{}` | +| `podSweeper.livenessProbe.enabled` | Enable livenessProbe on the podSweeper | `true` | +| `podSweeper.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `5` | +| `podSweeper.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `30` | +| `podSweeper.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `podSweeper.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `3` | +| `podSweeper.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `podSweeper.readinessProbe.enabled` | Enable readinessProbe on the podSweeper | `true` | +| `podSweeper.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `5` | +| `podSweeper.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `30` | +| `podSweeper.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `podSweeper.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `podSweeper.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `podSweeper.resources.limits` | The resources limits for the podSweeper container | `{}` | +| `podSweeper.resources.requests` | The requested resources for the podSweeper container | `{}` | +| `podSweeper.nodeSelector` | Node labels for pod assignment | `{}` | +| `podSweeper.tolerations` | Tolerations for podSweeper pod assignment. | `[]` | +| `podSweeper.affinity` | Affinity and anti-affinity for podSweeper pod assignment. | `{}` | +| `podSweeper.extraVolumeMounts` | Additional volumeMounts for podSweeper container(s). | `[]` | +| `podSweeper.extraVolumes` | Additional volumes for podSweeper pod(s). | `[]` | ### Server parameters @@ -86,8 +103,9 @@ | `server.replicaCount` | Number of server replicas | `1` | | `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | | `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | -| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.35.12-alpha` | +| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | | `server.podAnnotations` | Add extra annotations to the server pod | `{}` | +| `server.containerSecurityContext` | Security context for the container | `{}` | | `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | | `server.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | | `server.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | @@ -104,13 +122,13 @@ | `server.resources.requests` | The requested resources for the server container | `{}` | | `server.service.type` | The service type to use for the API server | `ClusterIP` | | `server.service.port` | The service port to expose the API server on | `8001` | -| `server.persistence.accessMode` | The access mode for the airbyte server pvc | `ReadWriteOnce` | -| `server.persistence.size` | The size of the pvc to use for the airbyte server pvc | `1Gi` | -| `server.persistence.storageClass` | The storage class to use for the airbyte server pvc | `""` | | `server.nodeSelector` | Node labels for pod assignment | `{}` | | `server.tolerations` | Tolerations for server pod assignment. | `[]` | +| `server.affinity` | Affinity and anti-affinity for server pod assignment. | `{}` | | `server.log.level` | The log level to log at | `INFO` | | `server.extraEnv` | Additional env vars for server pod(s). | `[]` | +| `server.extraVolumeMounts` | Additional volumeMounts for server container(s). | `[]` | +| `server.extraVolumes` | Additional volumes for server pod(s). | `[]` | ### Worker Parameters @@ -120,8 +138,9 @@ | `worker.replicaCount` | Number of worker replicas | `1` | | `worker.image.repository` | The repository to use for the airbyte worker image. | `airbyte/worker` | | `worker.image.pullPolicy` | the pull policy to use for the airbyte worker image | `IfNotPresent` | -| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.35.12-alpha` | +| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | | `worker.podAnnotations` | Add extra annotations to the worker pod(s) | `{}` | +| `worker.containerSecurityContext` | Security context for the container | `{}` | | `worker.livenessProbe.enabled` | Enable livenessProbe on the worker | `true` | | `worker.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | | `worker.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | @@ -138,74 +157,115 @@ | `worker.resources.requests` | The requested resources for the worker container | `{}` | | `worker.nodeSelector` | Node labels for pod assignment | `{}` | | `worker.tolerations` | Tolerations for worker pod assignment. | `[]` | +| `worker.affinity` | Affinity and anti-affinity for worker pod assignment. | `{}` | | `worker.log.level` | The log level to log at. | `INFO` | | `worker.extraEnv` | Additional env vars for worker pod(s). | `[]` | +| `worker.extraVolumeMounts` | Additional volumeMounts for worker container(s). | `[]` | +| `worker.extraVolumes` | Additional volumes for worker pod(s). | `[]` | ### Bootloader Parameters -| Name | Description | Value | -| ----------------------------- | -------------------------------------------------------------------- | -------------------- | -| `bootloader.image.repository` | The repository to use for the airbyte bootloader image. | `airbyte/bootloader` | -| `bootloader.image.pullPolicy` | the pull policy to use for the airbyte bootloader image | `IfNotPresent` | -| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.35.12-alpha` | +| Name | Description | Value | +| ------------------------------- | -------------------------------------------------------------------- | -------------------- | +| `bootloader.image.repository` | The repository to use for the airbyte bootloader image. | `airbyte/bootloader` | +| `bootloader.image.pullPolicy` | the pull policy to use for the airbyte bootloader image | `IfNotPresent` | +| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | +| `bootloader.podAnnotations` | Add extra annotations to the bootloader pod | `{}` | +| `bootloader.nodeSelector` | Node labels for pod assignment | `{}` | +| `bootloader.tolerations` | Tolerations for worker pod assignment. | `[]` | +| `bootloader.resources.limits` | The resources limits for the airbyte bootloader image | `{}` | +| `bootloader.resources.requests` | The requested resources for the airbyte bootloader image | `{}` | +| `bootloader.affinity` | Affinity and anti-affinity for bootloader pod assignment. | `{}` | ### Temporal parameters -| Name | Description | Value | -| --------------------------- | --------------------------------------------- | ----------------------- | -| `temporal.replicaCount` | The number of temporal replicas to deploy | `1` | -| `temporal.image.repository` | The temporal image repository to use | `temporalio/auto-setup` | -| `temporal.image.pullPolicy` | The pull policy for the temporal image | `IfNotPresent` | -| `temporal.image.tag` | The temporal image tag to use | `1.7.0` | -| `temporal.service.type` | The Kubernetes Service Type | `ClusterIP` | -| `temporal.service.port` | The temporal port and exposed kubernetes port | `7233` | -| `temporal.nodeSelector` | Node labels for pod assignment | `{}` | -| `temporal.tolerations` | Tolerations for pod assignment. | `[]` | -| `temporal.extraEnv` | Additional env vars for temporal pod(s). | `[]` | +| Name | Description | Value | +| --------------------------------------------- | ------------------------------------------------------- | ----------------------- | +| `temporal.replicaCount` | The number of temporal replicas to deploy | `1` | +| `temporal.image.repository` | The temporal image repository to use | `temporalio/auto-setup` | +| `temporal.image.pullPolicy` | The pull policy for the temporal image | `IfNotPresent` | +| `temporal.image.tag` | The temporal image tag to use | `1.7.0` | +| `temporal.service.type` | The Kubernetes Service Type | `ClusterIP` | +| `temporal.service.port` | The temporal port and exposed kubernetes port | `7233` | +| `temporal.podAnnotations` | Add extra annotations to the temporal pod | `{}` | +| `temporal.containerSecurityContext` | Security context for the container | `{}` | +| `temporal.extraInitContainers` | Additional InitContainers to initialize the pod | `[]` | +| `temporal.livenessProbe.enabled` | Enable livenessProbe on the temporal | `true` | +| `temporal.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `5` | +| `temporal.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `30` | +| `temporal.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `temporal.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `3` | +| `temporal.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `temporal.readinessProbe.enabled` | Enable readinessProbe on the temporal | `true` | +| `temporal.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `5` | +| `temporal.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `30` | +| `temporal.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `temporal.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `temporal.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `temporal.nodeSelector` | Node labels for temporal pod assignment | `{}` | +| `temporal.tolerations` | Tolerations for temporal pod assignment. | `[]` | +| `temporal.affinity` | Affinity and anti-affinity for temporal pod assignment. | `{}` | +| `temporal.extraEnv` | Additional env vars for temporal pod(s). | `[]` | +| `temporal.extraVolumeMounts` | Additional volumeMounts for temporal container(s). | `[]` | +| `temporal.extraVolumes` | Additional volumes for temporal pod(s). | `[]` | +| `temporal.resources.limits` | The resources limits for temporal pod(s) | `{}` | +| `temporal.resources.requests` | The requested resources for temporal pod(s) | `{}` | ### Airbyte Database parameters -| Name | Description | Value | -| -------------------------------------------- | ----------------------------------------------------------------------------------------- | ------------ | -| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | -| `postgresql.postgresqlUsername` | Airbyte Postgresql username | `airbyte` | -| `postgresql.postgresqlPassword` | Airbyte Postgresql password | `airbyte` | -| `postgresql.postgresqlDatabase` | Airbyte Postgresql database | `db-airbyte` | -| `postgresql.existingSecret` | Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) | `""` | -| `externalDatabase.host` | Database host | `localhost` | -| `externalDatabase.user` | non-root Username for Airbyte Database | `airbyte` | -| `externalDatabase.password` | Database password | `""` | -| `externalDatabase.existingSecret` | Name of an existing secret resource containing the DB password | `""` | -| `externalDatabase.existingSecretPasswordKey` | Name of an existing secret key containing the DB password | `""` | -| `externalDatabase.database` | Database name | `db-airbyte` | -| `externalDatabase.port` | Database port number | `5432` | +| Name | Description | Value | +| -------------------------------------------------- | ----------------------------------------------------------------------------------------- | ------------------------- | +| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | +| `postgresql.postgresqlUsername` | Airbyte Postgresql username | `airbyte` | +| `postgresql.postgresqlPassword` | Airbyte Postgresql password | `airbyte` | +| `postgresql.postgresqlDatabase` | Airbyte Postgresql database | `db-airbyte` | +| `postgresql.existingSecret` | Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) | `""` | +| `postgresql.containerSecurityContext.runAsNonRoot` | Ensures the container will run with a non-root user | `true` | +| `postgresql.commonAnnotations.helm.sh/hook` | It will determine when the hook should be rendered | `pre-install,pre-upgrade` | +| `postgresql.commonAnnotations.helm.sh/hook-weight` | The order in which the hooks are executed. If weight is lower, it has higher priority | `-1` | +| `externalDatabase.host` | Database host | `localhost` | +| `externalDatabase.user` | non-root Username for Airbyte Database | `airbyte` | +| `externalDatabase.password` | Database password | `""` | +| `externalDatabase.existingSecret` | Name of an existing secret resource containing the DB password | `""` | +| `externalDatabase.existingSecretPasswordKey` | Name of an existing secret key containing the DB password | `""` | +| `externalDatabase.database` | Database name | `db-airbyte` | +| `externalDatabase.port` | Database port number | `5432` | ### Logs parameters -| Name | Description | Value | -| ---------------------------- | ------------------------------------------------------ | ------------------ | -| `logs.accessKey.password` | Logs Access Key | `minio` | -| `logs.secretKey.password` | Logs Secret Key | `minio123` | -| `logs.minio.enabled` | Switch to enable or disable the Minio helm chart | `true` | -| `logs.externalMinio.enabled` | Switch to enable or disable an external Minio instance | `false` | -| `logs.externalMinio.host` | External Minio Host | `localhost` | -| `logs.externalMinio.port` | External Minio Port | `9000` | -| `logs.s3.enabled` | Switch to enable or disable custom S3 Log location | `false` | -| `logs.s3.bucket` | Bucket name where logs should be stored | `airbyte-dev-logs` | -| `logs.s3.bucketRegion` | Region of the bucket (must be empty if using minio) | `""` | -| `logs.gcs.bucket` | GCS bucket name | `""` | -| `logs.gcs.credentials` | The path the GCS creds are written to | `""` | +| Name | Description | Value | +| ---------------------------------- | ------------------------------------------------------ | ------------------ | +| `logs.accessKey.password` | Logs Access Key | `minio` | +| `logs.accessKey.existingSecret` | | `""` | +| `logs.accessKey.existingSecretKey` | | `""` | +| `logs.secretKey.password` | Logs Secret Key | `minio123` | +| `logs.secretKey.existingSecret` | | `""` | +| `logs.secretKey.existingSecretKey` | | `""` | +| `logs.minio.enabled` | Switch to enable or disable the Minio helm chart | `true` | +| `logs.externalMinio.enabled` | Switch to enable or disable an external Minio instance | `false` | +| `logs.externalMinio.host` | External Minio Host | `localhost` | +| `logs.externalMinio.port` | External Minio Port | `9000` | +| `logs.s3.enabled` | Switch to enable or disable custom S3 Log location | `false` | +| `logs.s3.bucket` | Bucket name where logs should be stored | `airbyte-dev-logs` | +| `logs.s3.bucketRegion` | Region of the bucket (must be empty if using minio) | `""` | +| `logs.gcs.bucket` | GCS bucket name | `""` | +| `logs.gcs.credentials` | The path the GCS creds are written to | `""` | +| `logs.gcs.credentialsJson` | Base64 encoded json GCP credentials file contents | `""` | ### Minio chart overwrites -| Name | Description | Value | -| -------------------------- | ---------------- | ---------- | -| `minio.accessKey.password` | Minio Access Key | `minio` | -| `minio.secretKey.password` | Minio Secret Key | `minio123` | - - +| Name | Description | Value | +| -------------------------------------------- | -------------------------------------------- | ---------- | +| `minio.accessKey.password` | Minio Access Key | `minio` | +| `minio.secretKey.password` | Minio Secret Key | `minio123` | +| `jobs.resources.limits` | The resources limits for jobs | `{}` | +| `jobs.resources.requests` | The requested resources for jobs | `{}` | +| `jobs.kube.annotations` | key/value annotations applied to kube jobs | `{}` | +| `jobs.kube.nodeSelector` | key/value node selector applied to kube jobs | `{}` | +| `jobs.kube.tolerations` | Tolerations for jobs.kube pod assignment. | `[]` | +| `jobs.kube.main_container_image_pull_secret` | image pull secret to use for job pod | `""` | diff --git a/charts/airbyte/airbyte-0.3.5.tgz b/charts/airbyte/airbyte-0.3.5.tgz new file mode 100644 index 0000000..52001ac Binary files /dev/null and b/charts/airbyte/airbyte-0.3.5.tgz differ diff --git a/charts/airbyte/airbyte-0.3.6.tgz b/charts/airbyte/airbyte-0.3.6.tgz new file mode 100644 index 0000000..903278f Binary files /dev/null and b/charts/airbyte/airbyte-0.3.6.tgz differ diff --git a/charts/airbyte/airbyte-0.3.7.tgz b/charts/airbyte/airbyte-0.3.7.tgz new file mode 100644 index 0000000..582c936 Binary files /dev/null and b/charts/airbyte/airbyte-0.3.7.tgz differ diff --git a/charts/airbyte/files/sweep-pod.sh b/charts/airbyte/files/sweep-pod.sh index 15422f2..1d5e56c 100644 --- a/charts/airbyte/files/sweep-pod.sh +++ b/charts/airbyte/files/sweep-pod.sh @@ -1,7 +1,9 @@ #!/bin/bash get_worker_pods () { - kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod --field-selector status.phase!=Running get pods -o go-template --template '{{range .items}} {{.metadata.name}} {{.status.phase}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \ + --field-selector status.phase!=Running get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime}{"\n"}{end}' } delete_worker_pod() { diff --git a/charts/airbyte/index.yaml b/charts/airbyte/index.yaml index f757277..12edebe 100644 --- a/charts/airbyte/index.yaml +++ b/charts/airbyte/index.yaml @@ -1,9 +1,81 @@ apiVersion: v1 entries: airbyte: + - apiVersion: v2 + appVersion: 0.39.23-alpha + created: "2022-06-23T13:31:27.013694-04:00" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x + - condition: postgresql.enabled + name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 10.x.x + - condition: logs.minio.enabled + name: minio + repository: https://charts.bitnami.com/bitnami + version: 11.x.x + description: Helm chart to deploy airbyte + digest: eb2303e0830f0afe3b6081da242dea73e78b25bc69c8835e8beb895a142f452a + name: airbyte + type: application + urls: + - airbyte-0.3.7.tgz + version: 0.3.7 + - apiVersion: v2 + appVersion: 0.39.18-alpha + created: "2022-06-23T13:31:26.985638-04:00" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x + - condition: postgresql.enabled + name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 10.x.x + - condition: logs.minio.enabled + name: minio + repository: https://charts.bitnami.com/bitnami + version: 11.x.x + description: Helm chart to deploy airbyte + digest: 0399945e94c80e55f8eb372ec685676d8e05d22c987b8d1838106cc19a51b1a0 + name: airbyte + type: application + urls: + - airbyte-0.3.6.tgz + version: 0.3.6 + - apiVersion: v2 + appVersion: 0.39.18-alpha + created: "2022-06-23T13:31:26.964901-04:00" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x + - condition: postgresql.enabled + name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 10.x.x + - condition: logs.minio.enabled + name: minio + repository: https://charts.bitnami.com/bitnami + version: 11.x.x + description: Helm chart to deploy airbyte + digest: 144075e759cbe852b56d423f47dd796823280db6498c2b795e902359f135bc17 + name: airbyte + type: application + urls: + - airbyte-0.3.5.tgz + version: 0.3.5 - apiVersion: v2 appVersion: 0.35.12-alpha - created: "2022-01-31T12:14:06.423369-05:00" + created: "2022-06-23T13:31:26.950665-04:00" dependencies: - name: common repository: https://charts.bitnami.com/bitnami @@ -27,7 +99,7 @@ entries: version: 0.3.1 - apiVersion: v2 appVersion: 0.35.12-alpha - created: "2022-01-31T12:14:06.414595-05:00" + created: "2022-06-23T13:31:26.936148-04:00" dependencies: - name: common repository: https://charts.bitnami.com/bitnami @@ -53,11 +125,10 @@ entries: - annotations: category: Infrastructure apiVersion: v2 - appVersion: 1.10.0 - created: "2022-01-31T12:14:06.423957-05:00" - description: A Library Helm Chart for grouping common logic between bitnami charts. - This chart is not deployable by itself. - digest: d13f6834adeb4d8d9cf94d2149680b08006543ab88944192f236decebd6313ae + appVersion: 1.16.0 + created: "2022-06-23T13:31:27.01597-04:00" + description: A Library Helm Chart for grouping common logic between bitnami charts. This chart is not deployable by itself. + digest: 612ba0ee47eecb204499b7021508c8ebff38d8b50899be717de3e44e7aa78aec home: https://github.com/bitnami/charts/tree/master/bitnami/common icon: https://bitnami.com/downloads/logos/bitnami-mark.png keywords: @@ -67,32 +138,30 @@ entries: - function - bitnami maintainers: - - email: containers@bitnami.com - name: Bitnami + - name: Bitnami + url: https://github.com/bitnami/charts name: common sources: - https://github.com/bitnami/charts - - http://www.bitnami.com/ + - https://www.bitnami.com/ type: library urls: - - charts/common-1.10.1.tgz - version: 1.10.1 + - charts/common-1.16.0.tgz + version: 1.16.0 minio: - annotations: category: Infrastructure apiVersion: v2 - appVersion: 2021.9.3 - created: "2022-01-31T12:14:06.426501-05:00" + appVersion: 2022.6.11 + created: "2022-06-23T13:31:27.02132-04:00" dependencies: - name: common repository: https://charts.bitnami.com/bitnami tags: - bitnami-common version: 1.x.x - description: Bitnami Object Storage based on MinIO® is an object storage server, - compatible with Amazon S3 cloud storage service, mainly used for storing unstructured - data (such as photos, videos, log files, etc.) - digest: 8eb8c9056b8c2c64570976ad7f51fa7b71486012c8f7ba965100b369fa58ff4b + description: MinIO(R) is an object storage server, compatible with Amazon S3 cloud storage service, mainly used for storing unstructured data (such as photos, videos, log files, etc.). + digest: 4a4370b34b3380aaa96663429d3c6bd970838855589e4310e6bce968ee9cc1e7 home: https://github.com/bitnami/charts/tree/master/bitnami/minio icon: https://bitnami.com/assets/stacks/minio/img/minio-stack-220x234.png keywords: @@ -102,28 +171,27 @@ entries: - s3 - cluster maintainers: - - email: containers@bitnami.com - name: Bitnami + - name: Bitnami + url: https://github.com/bitnami/charts name: minio sources: - https://github.com/bitnami/bitnami-docker-minio - https://min.io urls: - - charts/minio-7.3.2.tgz - version: 7.3.2 + - charts/minio-11.7.4.tgz + version: 11.7.4 postgresql: - annotations: category: Database apiVersion: v2 appVersion: 11.14.0 - created: "2022-01-31T12:14:06.429256-05:00" + created: "2022-06-23T13:31:27.025281-04:00" dependencies: - name: common repository: https://charts.bitnami.com/bitnami version: 1.x.x - description: Chart for PostgreSQL, an object-relational database management system - (ORDBMS) with an emphasis on extensibility and on standards-compliance. - digest: bf390d423e45c512228ad22c2e5899acfc420c0bb424a169ff8770ce3cb05495 + description: Chart for PostgreSQL, an object-relational database management system (ORDBMS) with an emphasis on extensibility and on standards-compliance. + digest: 6012be8e28cad57f87320c1b035b668c6d6d0325e384ce4732ec154e452dbca0 home: https://github.com/bitnami/charts/tree/master/bitnami/postgresql icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png keywords: @@ -143,6 +211,6 @@ entries: - https://github.com/bitnami/bitnami-docker-postgresql - https://www.postgresql.org/ urls: - - charts/postgresql-10.13.9.tgz - version: 10.13.9 -generated: "2022-01-31T12:14:06.403751-05:00" + - charts/postgresql-10.16.2.tgz + version: 10.16.2 +generated: "2022-06-23T13:31:26.920795-04:00" diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl index dd534d9..333a0c1 100644 --- a/charts/airbyte/templates/_helpers.tpl +++ b/charts/airbyte/templates/_helpers.tpl @@ -70,25 +70,21 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} - {{/* -Get the Postgresql credentials secret. +Get the Postgresql credentials secret name. */}} -{{- define "airbyte.postgresql.secretName" -}} -{{- if and (.Values.postgresql.enabled) (not .Values.postgresql.existingSecret) -}} - {{- printf "%s" (include "airbyte.postgresql.fullname" .) -}} -{{- else if and (.Values.postgresql.enabled) (.Values.postgresql.existingSecret) -}} - {{- printf "%s" .Values.postgresql.existingSecret -}} +{{- define "airbyte.database.secret.name" -}} +{{- if .Values.postgresql.enabled -}} + {{ template "postgresql.secretName" .Subcharts.postgresql }} {{- else }} {{- if .Values.externalDatabase.existingSecret -}} {{- printf "%s" .Values.externalDatabase.existingSecret -}} {{- else -}} - {{ printf "%s-%s" .Release.Name "externaldb" }} + {{ printf "%s-%s" (include "common.names.fullname" .) "secrets" }} {{- end -}} {{- end -}} {{- end -}} - {{/* Add environment variables to configure database values */}} @@ -111,9 +107,9 @@ Add environment variables to configure database values {{- end -}} {{/* -Add environment variables to configure database values +Get the Postgresql credentials secret password key */}} -{{- define "airbyte.database.existingsecret.key" -}} +{{- define "airbyte.database.secret.passwordKey" -}} {{- if .Values.postgresql.enabled -}} {{- printf "%s" "postgresql-password" -}} {{- else -}} @@ -124,7 +120,7 @@ Add environment variables to configure database values {{- printf "%s" "postgresql-password" -}} {{- end -}} {{- else -}} - {{- printf "%s" "postgresql-password" -}} + {{- printf "%s" "DATABASE_PASSWORD" -}} {{- end -}} {{- end -}} {{- end -}} @@ -162,7 +158,7 @@ Add environment variables to configure minio {{- if .Values.logs.minio.enabled -}} {{- printf "http://%s:%d" (include "airbyte.minio.fullname" .) 9000 -}} {{- else if .Values.logs.externalMinio.enabled -}} - {{- printf "http://%s:%d" .Values.logs.externalMinio.host .Values.logs.externalMinio.port -}} + {{- printf "http://%s:%g" .Values.logs.externalMinio.host .Values.logs.externalMinio.port -}} {{- else -}} {{- printf "" -}} {{- end -}} @@ -172,6 +168,17 @@ Add environment variables to configure minio {{- ternary "true" "" (or .Values.logs.minio.enabled .Values.logs.externalMinio.enabled) -}} {{- end -}} +{{/* +Returns the GCP credentials path +*/}} +{{- define "airbyte.gcpLogCredentialsPath" -}} +{{- if .Values.logs.gcs.credentialsJson }} + {{- printf "%s" "/secrets/gcs-log-creds/gcp.json" -}} +{{- else -}} + {{- printf "%s" .Values.logs.gcs.credentials -}} +{{- end -}} +{{- end -}} + {{/* Returns the Airbyte Scheduler Image */}} @@ -220,3 +227,25 @@ Returns the Temporal Image. TODO: This will probably be replaced if we move to u {{- define "airbyte.temporalImage" -}} {{- include "common.images.image" (dict "imageRoot" .Values.temporal.image "global" .Values.global) -}} {{- end -}} + +{{/* +Construct comma separated list of key/value pairs from object (useful for ENV var values) +*/}} +{{- define "airbyte.flattenMap" -}} +{{- $kvList := list -}} +{{- range $key, $value := . -}} +{{- $kvList = printf "%s=%s" $key $value | mustAppend $kvList -}} +{{- end -}} +{{ join "," $kvList }} +{{- end -}} + +{{/* +Construct semi-colon delimited list of comma separated key/value pairs from array of objects (useful for ENV var values) +*/}} +{{- define "airbyte.flattenArrayMap" -}} +{{- $mapList := list -}} +{{- range $element := . -}} +{{- $mapList = include "airbyte.flattenMap" $element | mustAppend $mapList -}} +{{- end -}} +{{ join ";" $mapList }} +{{- end -}} diff --git a/charts/airbyte/templates/bootloader/pod.yaml b/charts/airbyte/templates/bootloader/pod.yaml index e623818..59042ba 100644 --- a/charts/airbyte/templates/bootloader/pod.yaml +++ b/charts/airbyte/templates/bootloader/pod.yaml @@ -5,14 +5,24 @@ metadata: name: {{ include "common.names.fullname" . }}-bootloader labels: {{- include "airbyte.labels" . | nindent 4 }} - # Not ideal, however, otherwise there is no way to bootstrap server that uses sealed secrets: - # https://stackoverflow.com/questions/70929903/how-to-prevent-helm-from-renaming-the-resource-that-uses-hooks - # - # annotations: - # helm.sh/hook: pre-install,pre-upgrade - # helm.sh/hook-weight: "0" + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + {{- if .Values.bootloader.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.bootloader.podAnnotations "context" $) | nindent 4 }} + {{- end }} spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} restartPolicy: Never + {{- if .Values.bootloader.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.bootloader.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.bootloader.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.bootloader.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.bootloader.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.bootloader.affinity "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-bootloader-container image: {{ include "airbyte.bootloaderImage" . }} @@ -21,38 +31,31 @@ spec: - name: AIRBYTE_VERSION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: AIRBYTE_VERSION - name: DATABASE_HOST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_HOST - name: DATABASE_PORT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - - name: DATABASE_PASSWORD - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_PASSWORD - {{- else }} - name: DATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ include "airbyte.postgresql.secretName" . }} - key: {{ include "airbyte.database.existingsecret.key" . }} - {{- end }} + name: {{ include "airbyte.database.secret.name" . }} + key: {{ include "airbyte.database.secret.passwordKey" . }} - name: DATABASE_URL valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_URL - name: DATABASE_USER valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: DATABASE_USER + resources: {{- toYaml .Values.bootloader.resources | nindent 8 }} diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 32a4d7b..d626cf2 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -1,46 +1,51 @@ apiVersion: v1 kind: ConfigMap metadata: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "-1" data: AIRBYTE_VERSION: {{ .Values.version | default .Chart.AppVersion }} API_URL: {{ .Values.webapp.api.url }} - AWS_ACCESS_KEY_ID: {{ .Values.logs.accessKey.password }} - AWS_SECRET_ACCESS_KEY: {{ .Values.logs.secretKey.password }} CONFIG_ROOT: /configs - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.35.1.001" + CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.35.15.001" DATA_DOCKER_MOUNT: airbyte_data DATABASE_DB: {{ include "airbyte.database.name" . }} DATABASE_HOST: {{ include "airbyte.database.host" . }} - DATABASE_PASSWORD: {{ .Values.postgresql.postgresqlPassword }} DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} DATABASE_URL: {{ include "airbyte.database.url" . | quote }} - DATABASE_USER: {{ include "airbyte.database.user" . }} DB_DOCKER_MOUNT: airbyte_db FULLSTORY: {{ ternary "enabled" "disabled" .Values.webapp.fullstory.enabled }} GCS_LOG_BUCKET: {{ .Values.logs.gcs.bucket | quote }} - GOOGLE_APPLICATION_CREDENTIALS: {{ .Values.logs.gcs.credentials | quote }} + GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} INTERNAL_API_HOST: {{ include "common.names.fullname" . }}-server:{{ .Values.server.service.port }} IS_DEMO: {{ ternary "true" "false" .Values.webapp.isDemo | quote }} - JOB_MAIN_CONTAINER_CPU_LIMIT: "" - JOB_MAIN_CONTAINER_CPU_REQUEST: "" - JOB_MAIN_CONTAINER_MEMORY_LIMIT: "" - JOB_MAIN_CONTAINER_MEMORY_REQUEST: "" +{{- if $.Values.jobs.kube.annotations }} + JOB_KUBE_ANNOTATIONS: {{ $.Values.jobs.kube.annotations | include "airbyte.flattenMap" | quote }} +{{- end }} +{{- if $.Values.jobs.kube.nodeSelector }} + JOB_KUBE_NODE_SELECTORS: {{ $.Values.jobs.kube.nodeSelector | include "airbyte.flattenMap" | quote }} +{{- end }} +{{- if $.Values.jobs.kube.tolerations }} + JOB_KUBE_TOLERATIONS: {{ $.Values.jobs.kube.tolerations | include "airbyte.flattenArrayMap" | quote }} +{{- end }} + JOB_MAIN_CONTAINER_CPU_LIMIT: {{ ((.Values.jobs.resources | default dict).limits | default dict).cpu | default "" | quote }} + JOB_MAIN_CONTAINER_CPU_REQUEST: {{ ((.Values.jobs.resources | default dict).requests | default dict).cpu | default "" | quote }} + JOB_MAIN_CONTAINER_MEMORY_LIMIT: {{ ((.Values.jobs.resources | default dict).limits | default dict).memory | default "" | quote }} + JOB_MAIN_CONTAINER_MEMORY_REQUEST: {{ ((.Values.jobs.resources | default dict).requests | default dict).memory | default "" | quote }} +{{- if $.Values.jobs.kube.main_container_image_pull_secret }} + JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET: {{ $.Values.jobs.kube.main_container_image_pull_secret }} +{{- end }} JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.29.15.001" LOCAL_ROOT: /tmp/airbyte_local RUN_DATABASE_MIGRATION_ON_STARTUP: "true" - S3_LOG_BUCKET: {{ .Values.logs.s3.bucket }} + S3_LOG_BUCKET: {{ .Values.logs.s3.bucket | quote }} S3_LOG_BUCKET_REGION: {{ .Values.logs.s3.bucketRegion | quote }} S3_MINIO_ENDPOINT: {{ include "airbyte.minio.endpoint" . | quote }} S3_PATH_STYLE_ACCESS: {{ include "airbyte.s3PathStyleAccess" . | quote }} STATE_STORAGE_MINIO_BUCKET_NAME: airbyte-state-storage STATE_STORAGE_MINIO_ENDPOINT: {{ include "airbyte.minio.endpoint" . | quote }} - STATE_STORAGE_MINIO_ACCESS_KEY: {{ .Values.minio.accessKey.password | quote }} - STATE_STORAGE_MINIO_SECRET_ACCESS_KEY: {{ .Values.minio.secretKey.password | quote }} - SUBMITTER_NUM_THREADS: "10" TEMPORAL_HOST: {{ include "common.names.fullname" . }}-temporal:{{ .Values.temporal.service.port }} TEMPORAL_WORKER_PORTS: 9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040 TRACKING_STRATEGY: segment @@ -48,3 +53,9 @@ data: WORKER_ENVIRONMENT: kubernetes WORKSPACE_DOCKER_MOUNT: airbyte_workspace WORKSPACE_ROOT: /workspace + METRIC_CLIENT: "" + OTEL_COLLECTOR_ENDPOINT: "" + ACTIVITY_MAX_ATTEMPT: "" + ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" + ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" + WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: "" diff --git a/charts/airbyte/templates/gcs-log-creds-secret.yaml b/charts/airbyte/templates/gcs-log-creds-secret.yaml index 27e1f7a..077f1b5 100644 --- a/charts/airbyte/templates/gcs-log-creds-secret.yaml +++ b/charts/airbyte/templates/gcs-log-creds-secret.yaml @@ -1,7 +1,7 @@ -# TODO: Make this more configurable or support an existing secret apiVersion: v1 kind: Secret metadata: - name: gcs-log-creds + name: {{ include "common.names.fullname" . }}-gcs-log-creds +type: Opaque data: - gcp.json: "" + gcp.json: "{{ .Values.logs.gcs.credentialsJson }}" diff --git a/charts/airbyte/templates/pod-sweeper/configmap.yaml b/charts/airbyte/templates/pod-sweeper/configmap.yaml index cf19ce7..c1e14f9 100644 --- a/charts/airbyte/templates/pod-sweeper/configmap.yaml +++ b/charts/airbyte/templates/pod-sweeper/configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "airbyte.fullname" . }}-sweep-pod-script + name: {{ include "common.names.fullname" . }}-sweep-pod-script data: sweep-pod.sh: | {{- (.Files.Get "files/sweep-pod.sh") | nindent 4 }} diff --git a/charts/airbyte/templates/pod-sweeper/deployment.yaml b/charts/airbyte/templates/pod-sweeper/deployment.yaml index 731f109..ac0edfa 100644 --- a/charts/airbyte/templates/pod-sweeper/deployment.yaml +++ b/charts/airbyte/templates/pod-sweeper/deployment.yaml @@ -2,9 +2,11 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "airbyte.fullname" . }}-pod-sweeper + name: {{ include "common.names.fullname" . }}-pod-sweeper labels: {{- include "airbyte.labels" . | nindent 4 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/pod-sweeper/configmap.yaml") . | sha256sum }} spec: replicas: 1 selector: @@ -26,6 +28,9 @@ spec: {{- if .Values.podSweeper.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.tolerations "context" $) | nindent 8 }} {{- end }} + {{- if .Values.podSweeper.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.affinity "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-pod-sweeper image: {{ include "airbyte.podSweeperImage" . }} @@ -35,20 +40,55 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- if .Values.podSweeper.containerSecurityContext }} + securityContext: {{- toYaml .Values.podSweeper.containerSecurityContext | nindent 10 }} + {{- end }} volumeMounts: - mountPath: /script/sweep-pod.sh subPath: sweep-pod.sh name: sweep-pod-script - mountPath: /.kube name: kube-config + {{- if .Values.podSweeper.extraVolumeMounts }} + {{ toYaml .Values.podSweeper.extraVolumeMounts | nindent 8 }} + {{- end }} command: ["/bin/bash", "-c", /script/sweep-pod.sh] {{- if .Values.podSweeper.resources }} resources: {{- toYaml .Values.podSweeper.resources | nindent 10 }} {{- end }} + {{- if .Values.podSweeper.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - grep -aq sweep-pod.sh /proc/1/cmdline + initialDelaySeconds: {{ .Values.podSweeper.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.podSweeper.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.podSweeper.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.podSweeper.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.podSweeper.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.podSweeper.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - grep -aq sweep-pod.sh /proc/1/cmdline + initialDelaySeconds: {{ .Values.podSweeper.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.podSweeper.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.podSweeper.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.podSweeper.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.podSweeper.readinessProbe.failureThreshold }} + {{- end }} volumes: - name: kube-config emptyDir: {} - name: sweep-pod-script configMap: - name: {{ include "airbyte.fullname" . }}-sweep-pod-script + name: {{ include "common.names.fullname" . }}-sweep-pod-script defaultMode: 0755 + {{- if .Values.podSweeper.extraVolumes }} +{{ toYaml .Values.podSweeper.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte/templates/scheduler/deployment.yaml b/charts/airbyte/templates/scheduler/deployment.yaml deleted file mode 100644 index d49e3da..0000000 --- a/charts/airbyte/templates/scheduler/deployment.yaml +++ /dev/null @@ -1,202 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "airbyte.fullname" . }}-scheduler - labels: - {{- include "airbyte.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.scheduler.replicaCount }} - selector: - matchLabels: - airbyte: scheduler - template: - metadata: - labels: - airbyte: scheduler - {{- if .Values.scheduler.podAnnotations }} - annotations: - {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.podAnnotations "context" $) | nindent 8 }} - {{- end }} - spec: - {{- if .Values.scheduler.nodeSelector }} - nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.nodeSelector "context" $) | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.tolerations }} - tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.tolerations "context" $) | nindent 8 }} - {{- end }} - containers: - - name: airbyte-scheduler-container - image: {{ include "airbyte.schedulerImage" . }} - imagePullPolicy: "{{ .Values.scheduler.image.pullPolicy }}" - env: - - name: AIRBYTE_VERSION - valueFrom: - configMapKeyRef: - name: airbyte-env - key: AIRBYTE_VERSION - - name: CONFIG_ROOT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: CONFIG_ROOT - - name: DATABASE_HOST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_HOST - - name: DATABASE_PORT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - - name: DATABASE_PASSWORD - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_PASSWORD - {{- else }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "airbyte.postgresql.secretName" . }} - key: {{ include "airbyte.database.existingsecret.key" . }} - {{- end }} - - name: DATABASE_URL - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_URL - - name: DATABASE_USER - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_USER - - name: TRACKING_STRATEGY - valueFrom: - configMapKeyRef: - name: airbyte-env - key: TRACKING_STRATEGY - - name: WORKSPACE_DOCKER_MOUNT - value: workspace - - name: WORKSPACE_ROOT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: WORKSPACE_ROOT - - name: WORKER_ENVIRONMENT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: WORKER_ENVIRONMENT - - name: LOCAL_ROOT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: LOCAL_ROOT - - name: WEBAPP_URL - valueFrom: - configMapKeyRef: - name: airbyte-env - key: WEBAPP_URL - - name: TEMPORAL_HOST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: TEMPORAL_HOST - - name: TEMPORAL_WORKER_PORTS - valueFrom: - configMapKeyRef: - name: airbyte-env - key: TEMPORAL_WORKER_PORTS - - name: LOG_LEVEL - value: "{{ .Values.scheduler.log.level }}" - - name: JOB_KUBE_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SUBMITTER_NUM_THREADS - valueFrom: - configMapKeyRef: - name: airbyte-env - key: SUBMITTER_NUM_THREADS - - name: JOB_MAIN_CONTAINER_CPU_REQUEST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: JOB_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_MAIN_CONTAINER_CPU_LIMIT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: JOB_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: JOB_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - - name: S3_LOG_BUCKET - valueFrom: - configMapKeyRef: - name: airbyte-env - key: S3_LOG_BUCKET - - name: S3_LOG_BUCKET_REGION - valueFrom: - configMapKeyRef: - name: airbyte-env - key: S3_LOG_BUCKET_REGION - - name: AWS_ACCESS_KEY_ID - valueFrom: - configMapKeyRef: - name: airbyte-env - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - configMapKeyRef: - name: airbyte-env - key: AWS_SECRET_ACCESS_KEY - - name: S3_MINIO_ENDPOINT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: S3_MINIO_ENDPOINT - - name: S3_PATH_STYLE_ACCESS - valueFrom: - configMapKeyRef: - name: airbyte-env - key: S3_PATH_STYLE_ACCESS - - name: GOOGLE_APPLICATION_CREDENTIALS - valueFrom: - configMapKeyRef: - name: airbyte-env - key: GOOGLE_APPLICATION_CREDENTIALS - - name: GCS_LOG_BUCKET - valueFrom: - configMapKeyRef: - name: airbyte-env - key: GCS_LOG_BUCKET - - name: INTERNAL_API_HOST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: INTERNAL_API_HOST - {{- if .Values.scheduler.extraEnv }} - {{ .Values.scheduler.extraEnv | toYaml | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.resources }} - resources: {{- toYaml .Values.scheduler.resources | nindent 10 }} - {{- end }} - volumeMounts: - - name: gcs-log-creds-volume - mountPath: /secrets/gcs-log-creds - readOnly: true - volumes: - - name: gcs-log-creds-volume - secret: - secretName: gcs-log-creds diff --git a/charts/airbyte/templates/secret.yaml b/charts/airbyte/templates/secret.yaml new file mode 100644 index 0000000..92ba7a8 --- /dev/null +++ b/charts/airbyte/templates/secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }}-secrets + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +stringData: + AWS_ACCESS_KEY_ID: {{ .Values.logs.accessKey.password | quote }} + AWS_SECRET_ACCESS_KEY: {{ .Values.logs.secretKey.password | quote }} + {{ if and (not .Values.postgresql.enabled) (eq .Values.externalDatabase.existingSecret "") -}} + DATABASE_PASSWORD: {{ .Values.externalDatabase.password | quote }} + {{ end -}} + DATABASE_USER: {{ include "airbyte.database.user" . }} + STATE_STORAGE_MINIO_ACCESS_KEY: {{ .Values.minio.accessKey.password | quote }} + STATE_STORAGE_MINIO_SECRET_ACCESS_KEY: {{ .Values.minio.secretKey.password | quote }} diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml index 92b585d..e93e308 100644 --- a/charts/airbyte/templates/server/deployment.yaml +++ b/charts/airbyte/templates/server/deployment.yaml @@ -21,12 +21,16 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.server.podAnnotations "context" $) | nindent 8 }} {{- end }} spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} {{- if .Values.server.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.server.nodeSelector "context" $) | nindent 8 }} {{- end }} {{- if .Values.server.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.server.tolerations "context" $) | nindent 8 }} {{- end }} + {{- if .Values.server.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.server.affinity "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-server-container image: {{ include "airbyte.serverImage" . }} @@ -35,132 +39,140 @@ spec: - name: AIRBYTE_VERSION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: AIRBYTE_VERSION - name: CONFIG_ROOT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: CONFIG_ROOT - {{- if .Values.postgresql.enabled }} - - name: DATABASE_PASSWORD - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_PASSWORD - {{- else }} - name: DATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ include "airbyte.postgresql.secretName" . }} - key: {{ include "airbyte.database.existingsecret.key" . }} - {{- end }} + name: {{ include "airbyte.database.secret.name" . }} + key: {{ include "airbyte.database.secret.passwordKey" . }} - name: DATABASE_URL valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_URL - name: DATABASE_USER valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: DATABASE_USER - name: TRACKING_STRATEGY valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: TRACKING_STRATEGY - name: WORKER_ENVIRONMENT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: WORKER_ENVIRONMENT - name: WORKSPACE_ROOT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: WORKSPACE_ROOT - name: WEBAPP_URL valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: WEBAPP_URL - name: TEMPORAL_HOST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: TEMPORAL_HOST - name: LOG_LEVEL value: "{{ .Values.server.log.level }}" - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_CPU_REQUEST - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_CPU_LIMIT - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_MEMORY_REQUEST - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_LOG_BUCKET - name: S3_LOG_BUCKET_REGION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_LOG_BUCKET_REGION + {{- if and .Values.logs.accessKey.existingSecret .Values.logs.accessKey.existingSecretKey }} - name: AWS_ACCESS_KEY_ID valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ .Values.logs.accessKey.existingSecret }} + key: {{ .Values.logs.accessKey.existingSecretKey }} + {{- else }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: AWS_ACCESS_KEY_ID + {{- end }} + {{- if and .Values.logs.secretKey.existingSecret .Values.logs.secretKey.existingSecretKey }} - name: AWS_SECRET_ACCESS_KEY valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ .Values.logs.secretKey.existingSecret }} + key: {{ .Values.logs.secretKey.existingSecretKey }} + {{- else }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: AWS_SECRET_ACCESS_KEY + {{- end }} - name: S3_MINIO_ENDPOINT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_MINIO_ENDPOINT - name: S3_PATH_STYLE_ACCESS valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_PATH_STYLE_ACCESS - name: GOOGLE_APPLICATION_CREDENTIALS valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: GOOGLE_APPLICATION_CREDENTIALS - name: GCS_LOG_BUCKET valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: GCS_LOG_BUCKET - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION - name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION {{- if .Values.server.extraEnv }} {{ .Values.server.extraEnv | toYaml | nindent 8 }} @@ -194,20 +206,20 @@ spec: {{- if .Values.server.resources }} resources: {{- toYaml .Values.server.resources | nindent 10 }} {{- end }} + {{- if .Values.server.containerSecurityContext }} + securityContext: {{- toYaml .Values.server.containerSecurityContext | nindent 10 }} + {{- end }} volumeMounts: - - name: airbyte-data - mountPath: /configs - subPath: configs - - name: airbyte-data - mountPath: /workspace - subPath: workspace - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds readOnly: true + {{- if .Values.server.extraVolumeMounts }} + {{ toYaml .Values.server.extraVolumeMounts | nindent 8 }} + {{- end }} volumes: - - name: airbyte-data - persistentVolumeClaim: - claimName: {{ include "common.names.fullname" . }}-data - name: gcs-log-creds-volume secret: - secretName: gcs-log-creds + secretName: {{ include "common.names.fullname" . }}-gcs-log-creds + {{- if .Values.server.extraVolumes }} +{{ toYaml .Values.server.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte/templates/server/pvc-data.yaml b/charts/airbyte/templates/server/pvc-data.yaml deleted file mode 100644 index 62ef1e3..0000000 --- a/charts/airbyte/templates/server/pvc-data.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "common.names.fullname" . }}-data - labels: - airbyte: volume-data -spec: - accessModes: - - {{ .Values.server.persistence.accessMode | quote }} - resources: - requests: - storage: {{ .Values.server.persistence.size | quote }} - {{- include "common.storage.class" (dict "persistence" .Values.server.persistence "global" .Values.global) | nindent 2 }} diff --git a/charts/airbyte/templates/serviceaccount.yaml b/charts/airbyte/templates/serviceaccount.yaml index 2be26ab..5246ffa 100644 --- a/charts/airbyte/templates/serviceaccount.yaml +++ b/charts/airbyte/templates/serviceaccount.yaml @@ -5,8 +5,10 @@ metadata: name: {{ include "airbyte.serviceAccountName" . }} labels: {{- include "airbyte.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-10" + {{- with .Values.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} {{- end }} --- diff --git a/charts/airbyte/templates/temporal/configmap.yaml b/charts/airbyte/templates/temporal/configmap.yaml index bab965c..252ed29 100644 --- a/charts/airbyte/templates/temporal/configmap.yaml +++ b/charts/airbyte/templates/temporal/configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: airbyte-temporal-dynamicconfig + name: {{ include "common.names.fullname" . }}-temporal-dynamicconfig data: "development.yaml": | # when modifying, remember to update the docker-compose version of this file in temporal/dynamicconfig/development.yaml diff --git a/charts/airbyte/templates/temporal/deployment.yaml b/charts/airbyte/templates/temporal/deployment.yaml index f0ee8aa..f90d922 100644 --- a/charts/airbyte/templates/temporal/deployment.yaml +++ b/charts/airbyte/templates/temporal/deployment.yaml @@ -14,13 +14,25 @@ spec: metadata: labels: airbyte: temporal + {{- if .Values.temporal.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.temporal.podAnnotations "context" $) | nindent 8 }} + {{- end }} spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} {{- if .Values.temporal.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.nodeSelector "context" $) | nindent 8 }} {{- end }} {{- if .Values.temporal.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.tolerations "context" $) | nindent 8 }} {{- end }} + {{- if .Values.temporal.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.affinity "context" $) | nindent 8 }} + {{- end }} +{{- if .Values.temporal.extraInitContainers }} + initContainers: +{{- toYaml .Values.temporal.extraInitContainers | nindent 6 }} +{{- end }} containers: - name: airbyte-temporal image: {{ include "airbyte.temporalImage" . }} @@ -33,20 +45,15 @@ spec: - name: DB_PORT value: {{ include "airbyte.database.port" . | quote }} - name: POSTGRES_USER - value: {{ include "airbyte.database.user" . }} - {{- if .Values.postgresql.enabled }} - - name: POSTGRES_PWD valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_PASSWORD - {{- else }} + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets + key: DATABASE_USER - name: POSTGRES_PWD valueFrom: secretKeyRef: - name: {{ include "airbyte.postgresql.secretName" . }} - key: {{ include "airbyte.database.existingsecret.key" . }} - {{- end }} + name: {{ include "airbyte.database.secret.name" . }} + key: {{ include "airbyte.database.secret.passwordKey" . }} - name: POSTGRES_SEEDS value: {{ include "airbyte.database.host" . }} - name: DYNAMIC_CONFIG_FILE_PATH @@ -56,16 +63,51 @@ spec: {{- end }} ports: - containerPort: 7233 + {{- if .Values.temporal.containerSecurityContext }} + securityContext: {{- toYaml .Values.temporal.containerSecurityContext | nindent 10 }} + {{- end }} volumeMounts: - name: airbyte-temporal-dynamicconfig mountPath: "/etc/temporal/config/dynamicconfig/" + {{- if .Values.temporal.extraVolumeMounts }} + {{ toYaml .Values.temporal.extraVolumeMounts | nindent 8 }} + {{- end }} {{- if .Values.temporal.resources }} resources: {{- toYaml .Values.temporal.resources | nindent 10 }} {{- end }} + {{- if .Values.temporal.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - 'test $(ps -ef | grep -v grep | grep temporal-server | wc -l) -eq 1' + initialDelaySeconds: {{ .Values.temporal.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.temporal.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.temporal.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.temporal.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.temporal.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.temporal.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - 'test $(ps -ef | grep -v grep | grep temporal-server | wc -l) -eq 1' + initialDelaySeconds: {{ .Values.temporal.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.temporal.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.temporal.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.temporal.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.temporal.readinessProbe.failureThreshold }} + {{- end }} volumes: - name: airbyte-temporal-dynamicconfig configMap: - name: airbyte-temporal-dynamicconfig + name: {{ include "common.names.fullname" . }}-temporal-dynamicconfig items: - key: development.yaml path: development.yaml + {{- if .Values.temporal.extraVolumes }} +{{ toYaml .Values.temporal.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte/templates/tests/test-webapp.yaml b/charts/airbyte/templates/tests/test-webapp.yaml index 206aa49..823da96 100644 --- a/charts/airbyte/templates/tests/test-webapp.yaml +++ b/charts/airbyte/templates/tests/test-webapp.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: - name: "{{ include "airbyte.fullname" . }}-test-connection" + name: "{{ include "common.names.fullname" . }}-test-connection" labels: {{- include "airbyte.labels" . | nindent 4 }} annotations: diff --git a/charts/airbyte/templates/webapp/deployment.yaml b/charts/airbyte/templates/webapp/deployment.yaml index 1a74c07..f63ecbf 100644 --- a/charts/airbyte/templates/webapp/deployment.yaml +++ b/charts/airbyte/templates/webapp/deployment.yaml @@ -25,6 +25,9 @@ spec: {{- if .Values.webapp.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.tolerations "context" $) | nindent 8 }} {{- end }} + {{- if .Values.webapp.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.affinity "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-webapp-container image: {{ include "airbyte.webappImage" . }} @@ -33,36 +36,57 @@ spec: - name: AIRBYTE_VERSION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: AIRBYTE_VERSION - name: API_URL valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: API_URL - name: TRACKING_STRATEGY valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: TRACKING_STRATEGY - name: FULLSTORY valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: FULLSTORY - name: IS_DEMO valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: IS_DEMO - name: INTERNAL_API_HOST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: INTERNAL_API_HOST {{- if .Values.webapp.extraEnv }} {{ .Values.webapp.extraEnv | toYaml | nindent 8 }} {{- end }} + {{- if .Values.webapp.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: {{ .Values.webapp.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.webapp.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.webapp.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.webapp.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.webapp.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.webapp.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.webapp.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.webapp.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.webapp.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.webapp.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.webapp.readinessProbe.failureThreshold }} + {{- end }} ports: - name: http containerPort: 80 @@ -70,3 +94,14 @@ spec: {{- if .Values.webapp.resources }} resources: {{- toYaml .Values.webapp.resources | nindent 10 }} {{- end }} + {{- if .Values.webapp.containerSecurityContext }} + securityContext: {{- toYaml .Values.webapp.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.webapp.extraVolumeMounts }} + {{ toYaml .Values.webapp.extraVolumeMounts | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.webapp.extraVolumes }} +{{ toYaml .Values.webapp.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte/templates/webapp/ingress.yaml b/charts/airbyte/templates/webapp/ingress.yaml index 7a4342e..dec2ff3 100644 --- a/charts/airbyte/templates/webapp/ingress.yaml +++ b/charts/airbyte/templates/webapp/ingress.yaml @@ -1,5 +1,5 @@ --- -{{- if .Values.webapp.ingress.enabled -}} +{{ if .Values.webapp.ingress.enabled -}} {{- $fullName := include "airbyte.fullname" . -}} {{- $svcPort := .Values.webapp.service.port -}} {{- if and .Values.webapp.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} diff --git a/charts/airbyte/templates/webapp/service.yaml b/charts/airbyte/templates/webapp/service.yaml index 925360e..c769086 100644 --- a/charts/airbyte/templates/webapp/service.yaml +++ b/charts/airbyte/templates/webapp/service.yaml @@ -3,6 +3,10 @@ apiVersion: v1 kind: Service metadata: name: {{ include "common.names.fullname" . }}-webapp + {{- with .Values.webapp.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.webapp.service.type }} ports: diff --git a/charts/airbyte/templates/worker/deployment.yaml b/charts/airbyte/templates/worker/deployment.yaml index fd6c110..3a5febc 100644 --- a/charts/airbyte/templates/worker/deployment.yaml +++ b/charts/airbyte/templates/worker/deployment.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "airbyte.fullname" . }}-worker + name: {{ include "common.names.fullname" . }}-worker labels: {{- include "airbyte.labels" . | nindent 4 }} spec: @@ -26,6 +26,9 @@ spec: {{- if .Values.worker.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" $) | nindent 8 }} {{- end }} + {{- if .Values.worker.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.worker.affinity "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-worker-container image: {{ include "airbyte.workerImage" . }} @@ -34,82 +37,74 @@ spec: - name: AIRBYTE_VERSION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: AIRBYTE_VERSION - name: CONFIG_ROOT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: CONFIG_ROOT - name: DATABASE_HOST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_HOST - name: DATABASE_PORT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - - name: DATABASE_PASSWORD - valueFrom: - configMapKeyRef: - name: airbyte-env - key: DATABASE_PASSWORD - {{- else }} - name: DATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ include "airbyte.postgresql.secretName" . }} - key: {{ include "airbyte.database.existingsecret.key" . }} - {{- end }} + name: {{ include "airbyte.database.secret.name" . }} + key: {{ include "airbyte.database.secret.passwordKey" . }} - name: DATABASE_URL valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: DATABASE_URL - name: DATABASE_USER valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: DATABASE_USER - name: TRACKING_STRATEGY valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: TRACKING_STRATEGY - name: WORKSPACE_DOCKER_MOUNT value: workspace - name: WORKSPACE_ROOT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: WORKSPACE_ROOT - name: WORKER_ENVIRONMENT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: WORKER_ENVIRONMENT - name: LOCAL_ROOT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: LOCAL_ROOT - name: WEBAPP_URL valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: WEBAPP_URL - name: TEMPORAL_HOST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: TEMPORAL_HOST - name: TEMPORAL_WORKER_PORTS valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: TEMPORAL_WORKER_PORTS - name: LOG_LEVEL value: "{{ .Values.worker.log.level }}" @@ -117,96 +112,175 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: SUBMITTER_NUM_THREADS + {{- if $.Values.jobs.kube.annotations }} + - name: JOB_KUBE_ANNOTATIONS valueFrom: configMapKeyRef: name: airbyte-env - key: SUBMITTER_NUM_THREADS - - name: JOB_MAIN_CONTAINER_CPU_REQUEST + key: JOB_KUBE_ANNOTATIONS + {{- end }} + {{- if $.Values.jobs.kube.nodeSelector }} + - name: JOB_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: JOB_KUBE_NODE_SELECTORS + {{- end }} + {{- if $.Values.jobs.kube.tolerations }} + - name: JOB_KUBE_TOLERATIONS valueFrom: configMapKeyRef: name: airbyte-env + key: JOB_KUBE_TOLERATIONS + {{- end }} + {{- if $.Values.jobs.kube.main_container_image_pull_secret }} + - name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + {{- end }} + - name: JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_CPU_REQUEST - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_CPU_LIMIT - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_MEMORY_REQUEST - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_LOG_BUCKET - name: S3_LOG_BUCKET_REGION valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_LOG_BUCKET_REGION + {{- if and .Values.logs.accessKey.existingSecret .Values.logs.accessKey.existingSecretKey }} - name: AWS_ACCESS_KEY_ID valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ .Values.logs.accessKey.existingSecret }} + key: {{ .Values.logs.accessKey.existingSecretKey }} + {{- else }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: AWS_ACCESS_KEY_ID + {{- end }} + {{- if and .Values.logs.secretKey.existingSecret .Values.logs.secretKey.existingSecretKey }} - name: AWS_SECRET_ACCESS_KEY valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ .Values.logs.secretKey.existingSecret }} + key: {{ .Values.logs.secretKey.existingSecretKey }} + {{- else }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: AWS_SECRET_ACCESS_KEY + {{- end }} - name: S3_MINIO_ENDPOINT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_MINIO_ENDPOINT - name: S3_PATH_STYLE_ACCESS valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: S3_PATH_STYLE_ACCESS - name: GOOGLE_APPLICATION_CREDENTIALS valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: GOOGLE_APPLICATION_CREDENTIALS - name: GCS_LOG_BUCKET valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: GCS_LOG_BUCKET - name: STATE_STORAGE_MINIO_BUCKET_NAME valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: STATE_STORAGE_MINIO_BUCKET_NAME - name: STATE_STORAGE_MINIO_ACCESS_KEY valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: STATE_STORAGE_MINIO_ACCESS_KEY - name: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY valueFrom: - configMapKeyRef: - name: airbyte-env + secretKeyRef: + name: {{ include "common.names.fullname" . }}-secrets key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY - name: STATE_STORAGE_MINIO_ENDPOINT valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: STATE_STORAGE_MINIO_ENDPOINT - name: INTERNAL_API_HOST valueFrom: configMapKeyRef: - name: airbyte-env + name: {{ include "common.names.fullname" . }}-env key: INTERNAL_API_HOST + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: METRIC_CLIENT + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: METRIC_CLIENT + - name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: OTEL_COLLECTOR_ENDPOINT + - name: ACTIVITY_MAX_ATTEMPT + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: ACTIVITY_MAX_ATTEMPT + - name: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + - name: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + - name: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + valueFrom: + configMapKeyRef: + name: {{ include "common.names.fullname" . }}-env + key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS {{- if .Values.worker.extraEnv }} {{ .Values.worker.extraEnv | toYaml | nindent 8 }} {{- end }} @@ -268,11 +342,20 @@ spec: {{- if .Values.worker.resources }} resources: {{- toYaml .Values.worker.resources | nindent 10 }} {{- end }} + {{- if .Values.worker.containerSecurityContext }} + securityContext: {{- toYaml .Values.worker.containerSecurityContext | nindent 10 }} + {{- end }} volumeMounts: - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds readOnly: true + {{- if .Values.worker.extraVolumeMounts }} +{{ toYaml .Values.worker.extraVolumeMounts | nindent 8 }} + {{- end }} volumes: - name: gcs-log-creds-volume secret: - secretName: gcs-log-creds + secretName: {{ include "common.names.fullname" . }}-gcs-log-creds + {{- if .Values.worker.extraVolumes }} +{{ toYaml .Values.worker.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 2e08b9a..0a6a64c 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -1,11 +1,9 @@ ## @section Global Parameters ## @param global.imageRegistry Global Docker image registry -## @param global.storageClass Global StorageClass for Persistent Volume(s) ## global: imageRegistry: "" - storageClass: "" ## @section Common Parameters @@ -43,17 +41,59 @@ webapp: image: repository: airbyte/webapp pullPolicy: IfNotPresent - tag: 0.35.12-alpha + tag: 0.39.23-alpha ## @param webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) ## podAnnotations: {} + ## @param webapp.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + + ## Configure extra options for the webapp containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param webapp.livenessProbe.enabled Enable livenessProbe on the webapp + ## @param webapp.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param webapp.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param webapp.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param webapp.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param webapp.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param webapp.readinessProbe.enabled Enable readinessProbe on the webapp + ## @param webapp.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param webapp.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param webapp.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param webapp.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param webapp.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + ## @param webapp.service.type The service type to use for the webapp service ## @param webapp.service.port The service port to expose the webapp on + ## @param webapp.service.annotations Annotations for the webapp service resource service: type: ClusterIP port: 80 + annotations: {} ## Web app resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ @@ -85,6 +125,11 @@ webapp: ## tolerations: [] + ## @param webapp.affinity [object] Affinity and anti-affinity for webapp pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## Configure the ingress resource that allows you to access the Airbyte installation. ## ref: http://kubernetes.io/docs/user-guide/ingress/ ## @param webapp.ingress.enabled Set to true to enable ingress record generation @@ -128,65 +173,29 @@ webapp: ## value: "key=sample-value" extraEnv: [] -## @section Scheduler Parameters - -scheduler: - ## @param scheduler.replicaCount Number of scheduler replicas - replicaCount: 1 - - ## @param scheduler.image.repository The repository to use for the airbyte scheduler image. - ## @param scheduler.image.pullPolicy the pull policy to use for the airbyte scheduler image - ## @param scheduler.image.tag The airbyte scheduler image tag. Defaults to the chart's AppVersion - image: - repository: airbyte/scheduler - pullPolicy: IfNotPresent - tag: 0.35.12-alpha - - ## @param scheduler.podAnnotations [object] Add extra annotations to the scheduler pod + ## @param webapp.extraVolumeMounts [array] Additional volumeMounts for webapp container(s). + ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: var-run + ## mountPath: /var/run/ + ## - name: var-cache-nginx + ## mountPath: /var/cache/nginx + ## - mountPath: /etc/nginx/conf.d + ## name: nginx-conf-d ## - podAnnotations: {} - - ## Scheduler resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## We usually recommend not to specify default resources and to leave this as a conscious - ## choice for the user. This also increases chances charts run on environments with little - ## resources, such as Minikube. If you do want to specify resources, uncomment the following - ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. - ## @param scheduler.resources.limits [object] The resources limits for the scheduler container - ## @param scheduler.resources.requests [object] The requested resources for the scheduler container - resources: - ## Example: - ## limits: - ## cpu: 200m - ## memory: 1Gi - limits: {} - ## Examples: - ## requests: - ## memory: 256Mi - ## cpu: 250m - requests: {} - - ## @param scheduler.nodeSelector [object] Node labels for pod assignment - ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + extraVolumeMounts: [] + + ## @param webapp.extraVolumes [array] Additional volumes for webapp pod(s). + ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: var-run + ## emptyDir: {} + ## - name: var-cache-nginx + ## emptyDir: {} + ## - name: nginx-conf-d + ## emptyDir: {} ## - nodeSelector: {} - - ## @param scheduler.tolerations [array] Tolerations for scheduler pod assignment. - ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: [] - - ## @param scheduler.log.level The log level to log at. - log: - level: "INFO" - - ## @param scheduler.extraEnv [array] Additional env vars for scheduler pod(s). - ## Example: - ## - ## extraEnv: - ## - name: SAMPLE_ENV_VAR - ## value: "key=sample-value" - extraEnv: [] + extraVolumes: [] ## @section Pod Sweeper parameters @@ -203,6 +212,46 @@ podSweeper: ## podAnnotations: {} + ## @param podSweeper.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + + ## Configure extra options for the podSweeper containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param podSweeper.livenessProbe.enabled Enable livenessProbe on the podSweeper + ## @param podSweeper.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param podSweeper.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param podSweeper.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param podSweeper.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param podSweeper.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param podSweeper.readinessProbe.enabled Enable readinessProbe on the podSweeper + ## @param podSweeper.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param podSweeper.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param podSweeper.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param podSweeper.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param podSweeper.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + ## Pod Sweeper app resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## We usually recommend not to specify default resources and to leave this as a conscious @@ -233,6 +282,27 @@ podSweeper: ## tolerations: [] + ## @param podSweeper.affinity [object] Affinity and anti-affinity for podSweeper pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## @param podSweeper.extraVolumeMounts [array] Additional volumeMounts for podSweeper container(s). + ## Examples: + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## + extraVolumeMounts: [] + + ## @param podSweeper.extraVolumes [array] Additional volumes for podSweeper pod(s). + ## Examples: + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## + extraVolumes: [] + ## @section Server parameters server: @@ -245,12 +315,20 @@ server: image: repository: airbyte/server pullPolicy: IfNotPresent - tag: 0.35.12-alpha + tag: 0.39.23-alpha ## @param server.podAnnotations [object] Add extra annotations to the server pod ## podAnnotations: {} + ## @param server.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + ## Configure extra options for the server containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes ## @param server.livenessProbe.enabled Enable livenessProbe on the server @@ -309,20 +387,6 @@ server: type: ClusterIP port: 8001 - ## @param server.persistence.accessMode The access mode for the airbyte server pvc - ## @param server.persistence.size The size of the pvc to use for the airbyte server pvc - persistence: - size: 1Gi - accessMode: ReadWriteOnce - ## @param server.persistence.storageClass The storage class to use for the airbyte server pvc - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - storageClass: "" - ## @param server.nodeSelector [object] Node labels for pod assignment ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## @@ -333,6 +397,11 @@ server: ## tolerations: [] + ## @param server.affinity [object] Affinity and anti-affinity for server pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## @param server.log.level The log level to log at log: level: "INFO" @@ -345,6 +414,22 @@ server: ## value: "key=sample-value" extraEnv: [] + ## @param server.extraVolumeMounts [array] Additional volumeMounts for server container(s). + ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## + extraVolumeMounts: [] + + ## @param server.extraVolumes [array] Additional volumes for server pod(s). + ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## + extraVolumes: [] + ## @section Worker Parameters worker: @@ -357,12 +442,20 @@ worker: image: repository: airbyte/worker pullPolicy: IfNotPresent - tag: 0.35.12-alpha + tag: 0.39.23-alpha ## @param worker.podAnnotations [object] Add extra annotations to the worker pod(s) ## podAnnotations: {} + ## @param worker.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + ## Configure extra options for the worker containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes ## @param worker.livenessProbe.enabled Enable livenessProbe on the worker @@ -425,6 +518,11 @@ worker: ## tolerations: [] + ## @param worker.affinity [object] Affinity and anti-affinity for worker pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## @param worker.log.level The log level to log at. log: level: "INFO" @@ -437,6 +535,22 @@ worker: ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" extraEnv: [] + ## @param worker.extraVolumeMounts [array] Additional volumeMounts for worker container(s). + ## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## + extraVolumeMounts: [] + + ## @param worker.extraVolumes [array] Additional volumes for worker pod(s). + ## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## + extraVolumes: [] + ## @section Bootloader Parameters bootloader: @@ -446,7 +560,46 @@ bootloader: image: repository: airbyte/bootloader pullPolicy: IfNotPresent - tag: 0.35.12-alpha + tag: 0.39.23-alpha + + ## @param bootloader.podAnnotations [object] Add extra annotations to the bootloader pod + ## + podAnnotations: {} + + ## @param bootloader.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param bootloader.tolerations [array] Tolerations for worker pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Bootloader resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param bootloader.resources.limits [object] The resources limits for the airbyte bootloader image + ## @param bootloader.resources.requests [object] The requested resources for the airbyte bootloader image + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + + ## @param bootloader.affinity [object] Affinity and anti-affinity for bootloader pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} ## @section Temporal parameters ## TODO: Move to consuming temporal from a dedicated helm chart @@ -469,16 +622,80 @@ temporal: type: ClusterIP port: 7233 - ## @param temporal.nodeSelector [object] Node labels for pod assignment + ## @param temporal.podAnnotations [object] Add extra annotations to the temporal pod + ## + podAnnotations: {} + + ## @param temporal.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + + ## @param temporal.extraInitContainers Additional InitContainers to initialize the pod + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraInitContainers: + ## - name: config-loader + ## image: temporalio/auto-setup:1.7.0 + ## command: + ## - /bin/sh + ## - -c + ## - >- + ## find /etc/temporal/config/ -maxdepth 1 -mindepth 1 -exec cp -ar {} /config/ \; + ## volumeMounts: + ## - name: config + ## mountPath: /config + extraInitContainers: [] + + ## Configure extra options for the temporal containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param temporal.livenessProbe.enabled Enable livenessProbe on the temporal + ## @param temporal.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param temporal.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param temporal.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param temporal.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param temporal.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param temporal.readinessProbe.enabled Enable readinessProbe on the temporal + ## @param temporal.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param temporal.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param temporal.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param temporal.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param temporal.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param temporal.nodeSelector [object] Node labels for temporal pod assignment ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## nodeSelector: {} - ## @param temporal.tolerations [array] Tolerations for pod assignment. + ## @param temporal.tolerations [array] Tolerations for temporal pod assignment. ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [] + ## @param temporal.affinity [object] Affinity and anti-affinity for temporal pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## @param temporal.extraEnv [array] Additional env vars for temporal pod(s). ## Example: ## @@ -487,6 +704,46 @@ temporal: ## value: "key=sample-value" extraEnv: [] + ## @param temporal.extraVolumeMounts [array] Additional volumeMounts for temporal container(s). + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## - name: config + ## mountPath: /etc/temporal/config + ## + extraVolumeMounts: [] + + ## @param temporal.extraVolumes [array] Additional volumes for temporal pod(s). + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## - name: config + ## emptyDir: {} + ## + extraVolumes: [] + + ## Temporal resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param temporal.resources.limits [object] The resources limits for temporal pod(s) + ## @param temporal.resources.requests [object] The requested resources for temporal pod(s) + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## @section Airbyte Database parameters ## PostgreSQL chart configuration @@ -496,6 +753,9 @@ temporal: ## @param postgresql.postgresqlPassword Airbyte Postgresql password ## @param postgresql.postgresqlDatabase Airbyte Postgresql database ## @param postgresql.existingSecret Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) +## @param postgresql.containerSecurityContext.runAsNonRoot Ensures the container will run with a non-root user +## @param postgresql.commonAnnotations.helm.sh/hook It will determine when the hook should be rendered +## @param postgresql.commonAnnotations.helm.sh/hook-weight The order in which the hooks are executed. If weight is lower, it has higher priority ## postgresql: enabled: true @@ -505,6 +765,8 @@ postgresql: ## This secret is used in case of postgresql.enabled=true and we would like to specify password for newly created postgresql instance ## existingSecret: "" + containerSecurityContext: + runAsNonRoot: true commonAnnotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "-1" @@ -530,11 +792,19 @@ externalDatabase: ## @section Logs parameters logs: ## @param logs.accessKey.password Logs Access Key - ## @param logs.secretKey.password Logs Secret Key + ## @param logs.accessKey.existingSecret + ## @param logs.accessKey.existingSecretKey accessKey: password: minio + existingSecret: "" + existingSecretKey: "" + ## @param logs.secretKey.password Logs Secret Key + ## @param logs.secretKey.existingSecret + ## @param logs.secretKey.existingSecretKey secretKey: password: minio123 + existingSecret: "" + existingSecretKey: "" ## @param logs.minio.enabled Switch to enable or disable the Minio helm chart minio: @@ -559,9 +829,14 @@ logs: ## Google Cloud Storage (GCS) Log Location Configuration ## @param logs.gcs.bucket GCS bucket name ## @param logs.gcs.credentials The path the GCS creds are written to + ## @param logs.gcs.credentialsJson Base64 encoded json GCP credentials file contents gcs: bucket: "" + # If you are mounting an existing secret to extraVolumes on scheduler, server and worker + # deployments, then set credentials to the path of the mounted JSON file credentials: "" + # If credentialsJson is set then credentials auto resolves (to /secrets/gcs-log-creds/gcp.json) + credentialsJson: "" ## @section Minio chart overwrites ## @param minio.accessKey.password Minio Access Key @@ -571,3 +846,50 @@ minio: password: minio secretKey: password: minio123 + +jobs: + ## Jobs resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param jobs.resources.limits [object] The resources limits for jobs + ## @param jobs.resources.requests [object] The requested resources for jobs + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + + kube: + ## JOB_KUBE_ANNOTATIONS + ## pod annotations of the sync job and the default pod annotations fallback for others jobs + ## @param jobs.kube.annotations [object] key/value annotations applied to kube jobs + annotations: {} + + ## JOB_KUBE_NODE_SELECTORS + ## pod node selector of the sync job and the default pod node selector fallback for others jobs + ## @param jobs.kube.nodeSelector [object] key/value node selector applied to kube jobs + nodeSelector: {} + + ## JOB_KUBE_TOLERATIONS + ## @param jobs.kube.tolerations [array] Tolerations for jobs.kube pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## any boolean values should be quoted to ensure the value is passed through as a string, eg: + ## - key: airbyte-server + ## operator: Equal + ## value: "true" + ## effect: NoSchedule + tolerations: [] + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + ## image pull secret to use for job pod + ## @param jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod + main_container_image_pull_secret: "" diff --git a/charts/index.yaml b/charts/index.yaml new file mode 100644 index 0000000..63ff09a --- /dev/null +++ b/charts/index.yaml @@ -0,0 +1,3 @@ +apiVersion: v1 +entries: {} +generated: "2022-06-15T11:38:53.438126-04:00"