chore: Add the preprod refresh versions script

MAP-1975

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml

@@ -1,5 +1,5 @@
 {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "create-temp-tables" | trunc 52 }}
-{{- if .Values.scripts.preprodRefresh.enabled }}
+{{- if .Values.scripts.preprodRefresh.main.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml

@@ -1,5 +1,5 @@
 {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "copy-from-temp-sql" | trunc 52 }}
-{{- if .Values.scripts.preprodRefresh.enabled }}
+{{- if .Values.scripts.preprodRefresh.main.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml

@@ -1,5 +1,5 @@
 {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "drop-temp-tables" | trunc 52 }}
-{{- if .Values.scripts.preprodRefresh.enabled }}
+{{- if .Values.scripts.preprodRefresh.main.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml

@@ -1,5 +1,5 @@
 {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-script" | trunc 52 }}
-{{- if .Values.scripts.preprodRefresh.enabled }}
+{{- if .Values.scripts.preprodRefresh.main.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml

@@ -4,14 +4,14 @@
 {{- $script2 := printf "%s-%s" (include "generic-service.fullname" $) "copy-from-temp-sql" | trunc 52 }}
 {{- $script3 := printf "%s-%s" (include "generic-service.fullname" $) "drop-temp-tables" | trunc 52 }}
 {{- $script4 := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-script" | trunc 52 }}
-{{- if .Values.scripts.preprodRefresh.enabled }}
+{{- if .Values.scripts.preprodRefresh.main.enabled }}
 ---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: {{ $fullName }}
 spec:
-  schedule: {{ .Values.scripts.preprodRefresh.schedule }}
+  schedule: {{ .Values.scripts.preprodRefresh.main.schedule }}
   concurrencyPolicy: "Forbid"
   successfulJobsHistoryLimit: 5
   failedJobsHistoryLimit: 3

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml

@@ -0,0 +1,44 @@
+{{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-versions-script" | trunc 52 }}
+{{- if .Values.scripts.preprodRefresh.versions.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $fullName }}
+data:
+  entrypoint.sh: |-
+    #!/bin/bash
+    from=$(date --date="26 hours ago" '+%Y-%m-%d %T')
+    to=$(date '+%Y-%m-%d %T')
+    echo "Refreshing versions from ${from} to ${to}" 
+        
+    
+    execDest() {
+      cmd=$1
+      psql $DEST_DB_INSTANCE -c "${cmd}"
+    }
+  
+    execSource() {
+      cmd=$1
+      psql $SOURCE_DB_INSTANCE -c "${cmd}"
+    }
+    
+    
+    execSource "DROP TABLE IF EXISTS versions_temp CASCADE;"
+    execSource "CREATE TABLE versions_temp (LIKE versions);"
+    execDest "DROP TABLE IF EXISTS versions_temp CASCADE;"
+    execDest "CREATE TABLE versions_temp (LIKE versions);"
+    
+    echo "Copying versions to source temp table"
+    execSource "INSERT INTO versions_temp SELECT * FROM versions where created_at >= '${from}' and created_at < '${to}';"
+    
+    echo "Starting dump of data to preprod"
+    pg_dump $SOURCE_DB_INSTANCE -t "versions_temp" -O --section=data | psql $DEST_DB_INSTANCE 
+    echo "Finished dumping data"
+
+    echo "Copying data from destination temp table to actual table"    
+    execDest "INSERT INTO versions (id, item_type, item_id, event, whodunnit, object, created_at, supplier_id)  SELECT nextval('versions_id_seq'::regclass), item_type, item_id, event, whodunnit, object, created_at, supplier_id FROM versions_temp ON CONFLICT (item_id, item_type, created_at) DO NOTHING;"
+    echo "Dropping temporary tables"
+    execDest "DROP TABLE IF EXISTS versions_temp CASCADE;"
+    execSource "DROP TABLE IF EXISTS versions_temp CASCADE;"
+    echo "All done!"
+{{- end }}

helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml

@@ -0,0 +1,52 @@
+{{- $genericService := index .Values "generic-service" -}}
+{{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh" | trunc 52 }}
+{{- $script1 := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-versions-script" | trunc 52 }}
+{{- if .Values.scripts.preprodRefresh.versions.enabled }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: preprod-refresh-versions-job
+spec:
+  schedule: {{ .Values.scripts.preprodRefresh.versions.schedule }}
+  concurrencyPolicy: "Forbid"
+  successfulJobsHistoryLimit: 5
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 345600
+      backoffLimit: 0
+      activeDeadlineSeconds: 7200
+      template:
+        spec:
+          serviceAccountName: "book-a-secure-move-api"
+          containers:
+            - name: preprod-versions-refresh
+              image: "ghcr.io/ministryofjustice/hmpps-devops-tools:latest"
+              command:
+                - /bin/entrypoint.sh
+              volumeMounts:
+                - name: preprod-refresh-versions-script
+                  mountPath: /bin/entrypoint.sh
+                  readOnly: true
+                  subPath: entrypoint.sh
+              env:
+                - name: DEST_DB_INSTANCE
+                  valueFrom:
+                    secretKeyRef:
+                      name: preprod-rds-creds
+                      key: url
+                - name: SOURCE_DB_INSTANCE
+                  valueFrom:
+                    secretKeyRef:
+                      name: rds-instance-hmpps-book-secure-move-api-production
+                      key: url
+                - name: AWS_DEFAULT_REGION
+                  value: "eu-west-2"
+          restartPolicy: "Never"
+          volumes:
+            - name: preprod-refresh-versions-script
+              configMap:
+                name: {{ $script1 }}
+                defaultMode: 0755
+{{- end }}

helm_deploy/hmpps-book-secure-move-api/values.yaml

@@ -62,8 +62,14 @@ dashboards:
 
 scripts:
   preprodRefresh:
-    enabled: false
-    # schedule: "15 6 * * *"
+    main:
+      enabled: false
+      # schedule: "0 */4 * * *"
+      # dataSince: "6 hours ago"
+    versions:
+      enabled: false
+      # schedule: "15 7 * * *"
+      # dataSince: "26 hours ago"
   reports:
     enabled: false
     dailyIpt:

helm_deploy/values-production.yaml

@@ -119,8 +119,14 @@ dashboards:
 
 scripts:
   preprodRefresh:
-    enabled: true
-    schedule: "15 6 * * *"
+    main:
+      enabled: true
+      schedule: "0 */4 * * *"
+      dataSince: "6 hours ago"
+    versions:
+      enabled: true
+      schedule: "15 7 * * *"
+      dataSince: "26 hours ago"
   reports:
     enabled: true
     dailyIpt: