helm chart for cht sync

dbt/dbt-run.py

@@ -28,7 +28,17 @@ def connection():
     with conn.cursor() as cur:
         cur.execute(f"""
             CREATE SCHEMA IF NOT EXISTS 
-            {os.getenv('POSTGRES_SCHEMA')}
+            {os.getenv('POSTGRES_SCHEMA')};
+
+            CREATE TABLE IF NOT EXISTS {os.getenv('POSTGRES_SCHEMA')}.{os.getenv('POSTGRES_TABLE')} (
+                "@version" TEXT,
+                "@timestamp" TIMESTAMP,
+                "_id" TEXT,
+                "_rev" TEXT,
+                doc jsonb,
+                doc_as_upsert BOOLEAN,
+                UNIQUE ("_id", "_rev")
+            );
         """)
     conn.commit()
 

deploy/cht_sync/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: cht_sync
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.1.0"

deploy/cht_sync/templates/dbt.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cht-sync-dbt
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cht-sync
+  template:
+    metadata:
+      labels:
+        app: cht-sync
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: dbt
+          image: medicmobile/dataemon:{{ .Values.image_tag | default "latest" }}
+          resources:
+            limits:
+              cpu: {{ (.Values.dbt).cpu_limit | default "500m" }}
+              memory: {{ (.Values.dbt).memory_limit | default "1Gi" }}
+          env:
+            - name: POSTGRES_HOST
+              value: {{ .Values.postgres.host | default "postgres" }}
+            - name: POSTGRES_USER
+              value: {{ .Values.postgres.user }}
+            - name: POSTGRES_PORT
+              value: {{ .Values.postgres.port | default "5432" | quote }}
+            - name: POSTGRES_PASSWORD
+              value: {{ .Values.postgres.password }}
+            - name: POSTGRES_DB
+              value: {{ .Values.postgres.db }}
+            - name: POSTGRES_TABLE
+              value: {{ .Values.postgres.table }}
+            - name: POSTGRES_SCHEMA
+              value: {{ .Values.postgres.schema }}
+            - name: ROOT_POSTGRES_SCHEMA
+              value: {{ .Values.postgres.schema }}
+            - name: CHT_PIPELINE_BRANCH_URL
+              value: {{ .Values.cht_pipeline_branch_url }}
+

deploy/cht_sync/templates/logstash.yaml

@@ -0,0 +1,45 @@
+# templates/logstash.yaml
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cht-sync-logstash
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cht-sync
+  template:
+    metadata:
+      labels:
+        app: cht-sync
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: logstash
+          image: medicmobile/cht-logstash-couchdb:{{ .Values.image_tag | default "latest" }}
+          resources:
+            limits:
+              cpu: {{ (.Values.logstash).cpu_limit | default "2" }}
+              memory: {{ (.Values.logstash).memory_limit | default "2Gi" }}
+          env:
+            - name: COUCHDB_USER
+              value: {{ .Values.couchdb.user }}
+            - name: COUCHDB_PASSWORD
+              value: {{ .Values.couchdb.password }}
+            - name: COUCHDB_HOST
+              value: {{ .Values.couchdb.host }}
+            - name: COUCHDB_DBS
+              value: {{ .Values.couchdb.dbs }}
+            - name: COUCHDB_PORT
+              value: {{ .Values.couchdb.port | quote }}
+            - name: COUCHDB_SECURE
+              value: {{ .Values.couchdb.secure | default "false" | quote }}
+            - name: COUCHDB_SEQ # don't allow this to be changed
+              value: "/tmp/couchdb/sequence_path.txt"
+            - name: REDIS_HOST
+              value: "redis"
+            - name: REDIS_PORT
+              value: "6379"
+            - name: REDIS_KEY
+              value: {{ (.Values.redis).key | default "couchdb" }}

deploy/cht_sync/templates/postgres-service.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.postgres.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  selector:
+    inner.service: postgres
+  ports:
+    - protocol: TCP
+      port: {{ .Values.postgres.port | default "5432" }}
+      targetPort: 5432
+{{- end }}

deploy/cht_sync/templates/postgres.yaml

@@ -0,0 +1,47 @@
+# templates/postgres.yaml
+
+{{- if .Values.postgres.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cht-sync-postgres
+spec:
+  serviceName: postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cht-sync
+      inner.service: postgres
+  template:
+    metadata:
+      labels:
+        app: cht-sync
+        inner.service: postgres
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: postgres-c
+          image: postgres:13
+          resources:
+            limits:
+              cpu: {{ (.Values.postgres).cpu_limit | default "1000m" }}
+              memory: {{ (.Values.postgres).memory_limit | default "2Gi" }}
+          env:
+            - name: POSTGRES_USER
+              value: {{ .Values.postgres.user }}
+            - name: POSTGRES_PASSWORD
+              value: {{ .Values.postgres.password }}
+            - name: POSTGRES_DB
+              value: {{ .Values.postgres.db }}
+          ports:
+            - containerPort: {{ .Values.postgres.port | default "5432" }}
+
+  volumeClaimTemplates:
+    - metadata:
+        name: postgres-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ .Values.postgres.storageSize | default "1Gi" }}
+{{- end }}

deploy/cht_sync/templates/postgrest-service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgrest
+spec:
+  selector:
+    inner.service: postgrest
+  ports:
+    - protocol: TCP
+      port: 3000

deploy/cht_sync/templates/postgrest.yaml

@@ -0,0 +1,39 @@
+# templates/postgrest.yaml
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cht-sync-postgrest
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cht-sync
+      inner.service: postgrest
+  template:
+    metadata:
+      labels:
+        app: cht-sync
+        inner.service: postgrest
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: postgrest
+          image: postgrest/postgrest
+          resources:
+            limits:
+              cpu: {{ (.Values.postgrest).cpu_limit | default "1000m" }}
+              memory: {{ (.Values.postgrest).memory_limit | default "500Mi" }}
+          env:
+            - name: PGRST_DB_URI
+              value: |
+                {{ printf "postgres://%s:%s@%s:%d/%s"
+                .Values.postgres.user
+                .Values.postgres.password
+                ( .Values.postgres.host | default "postgres" )
+                (int .Values.postgres.port | default 5432)
+                .Values.postgres.db }}
+            - name: PGRST_DB_SCHEMA
+              value: {{ .Values.postgres.schema }}
+            - name: PGRST_DB_ANON_ROLE
+              value: {{ .Values.postgres.user }}

deploy/cht_sync/templates/redis-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  selector:
+    inner.service: redis
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379

deploy/cht_sync/templates/redis-worker.yaml

@@ -0,0 +1,35 @@
+# templates/postgrest.yaml
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cht-sync-redis-worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cht-sync
+  template:
+    metadata:
+      labels:
+        app: cht-sync
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: redis-worker
+          image: dockermedic/cht-sync-redis-worker:{{ .Values.image_tag | default "latest" }}
+          resources:
+            limits:
+              cpu: {{ (.Values.redis_worker).cpu_limit | default "500m" }}
+              memory: {{ (.Values.redis_worker).memory_limit | default "500Mi" }}
+          env:
+            - name: REDIS_HOST
+              value: "redis"
+            - name: REDIS_PORT
+              value: "6379"
+            - name: REDIS_KEY
+              value: {{ (.Values.redis).key | default "couchdb" }}
+            - name: POSTGREST_ENDPOINT
+              value: "postgrest:3000"
+            - name: REDIS_BATCH_SIZE
+              value: {{ (.Values.redis).batch_size | default "100" | quote }}

deploy/cht_sync/templates/redis.yaml

@@ -0,0 +1,35 @@
+# templates/postgrest.yaml
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cht-sync-redis
+spec:
+  serviceName: redis
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cht-sync
+      inner.service: redis
+  template:
+    metadata:
+      labels:
+        app: cht-sync
+        inner.service: redis
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: redis
+          image: redis:latest
+          resources:
+            limits:
+              cpu: {{ (.Values.redis).cpu_limit | default "500m" }}
+              memory: {{ (.Values.redis).memory_limit | default "2Gi" }}
+  volumeClaimTemplates:
+    - metadata:
+        name: redis-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ (.Values.redis).storageSize | default "1Gi" }}

deploy/cht_sync/values.yaml.template

@@ -0,0 +1,25 @@
+
+postgres:
+  enabled: true  # if true a postgres service will not be created in this cluster
+  user: ""
+  password: ""
+  db: ""
+  schema: "v1"
+  table: "medic"  
+  host: "postgres" # if postgres is outside the cluster
+  port: 5432 # if postgres is outside the cluster
+
+cht_pipeline_branch_url: "https://github.com/medic/cht-pipeline.git#main"
+
+couchdb:
+  user: ""
+  password: ""
+  dbs: "medic" # space separated list of databases you want to sync e.g "medic medic-sentinel"
+  host: ""
+  port: "5984"
+  secure: "false"
+
+# redis
+redis:
+  key: "couchdb"
+  batch_size: 1000