diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml index df3de0826..bd6c545dd 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "create-temp-tables" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml index 62cfad232..834ecaaa2 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "copy-from-temp-sql" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml index 64c5a9542..496e813a5 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "drop-temp-tables" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml index 466bf207b..4528756cf 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-script" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml index f56183b40..d8f45689f 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml @@ -4,14 +4,14 @@ {{- $script2 := printf "%s-%s" (include "generic-service.fullname" $) "copy-from-temp-sql" | trunc 52 }} {{- $script3 := printf "%s-%s" (include "generic-service.fullname" $) "drop-temp-tables" | trunc 52 }} {{- $script4 := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-script" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} --- apiVersion: batch/v1 kind: CronJob metadata: name: {{ $fullName }} spec: - schedule: {{ .Values.scripts.preprodRefresh.schedule }} + schedule: {{ .Values.scripts.preprodRefresh.main.schedule }} concurrencyPolicy: "Forbid" successfulJobsHistoryLimit: 5 failedJobsHistoryLimit: 3 diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml new file mode 100644 index 000000000..1218380b9 --- /dev/null +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml @@ -0,0 +1,44 @@ +{{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-versions-script" | trunc 52 }} +{{- if .Values.scripts.preprodRefresh.versions.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $fullName }} +data: + entrypoint.sh: |- + #!/bin/bash + from=$(date --date="26 hours ago" '+%Y-%m-%d %T') + to=$(date '+%Y-%m-%d %T') + echo "Refreshing versions from ${from} to ${to}" + + + execDest() { + cmd=$1 + psql $DEST_DB_INSTANCE -c "${cmd}" + } + + execSource() { + cmd=$1 + psql $SOURCE_DB_INSTANCE -c "${cmd}" + } + + + execSource "DROP TABLE IF EXISTS versions_temp CASCADE;" + execSource "CREATE TABLE versions_temp (LIKE versions);" + execDest "DROP TABLE IF EXISTS versions_temp CASCADE;" + execDest "CREATE TABLE versions_temp (LIKE versions);" + + echo "Copying versions to source temp table" + execSource "INSERT INTO versions_temp SELECT * FROM versions where created_at >= '${from}' and created_at < '${to}';" + + echo "Starting dump of data to preprod" + pg_dump $SOURCE_DB_INSTANCE -t "versions_temp" -O --section=data | psql $DEST_DB_INSTANCE + echo "Finished dumping data" + + echo "Copying data from destination temp table to actual table" + execDest "INSERT INTO versions (id, item_type, item_id, event, whodunnit, object, created_at, supplier_id) SELECT nextval('versions_id_seq'::regclass), item_type, item_id, event, whodunnit, object, created_at, supplier_id FROM versions_temp ON CONFLICT (item_id, item_type, created_at) DO NOTHING;" + echo "Dropping temporary tables" + execDest "DROP TABLE IF EXISTS versions_temp CASCADE;" + execSource "DROP TABLE IF EXISTS versions_temp CASCADE;" + echo "All done!" +{{- end }} diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml new file mode 100644 index 000000000..2025edd9a --- /dev/null +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml @@ -0,0 +1,52 @@ +{{- $genericService := index .Values "generic-service" -}} +{{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh" | trunc 52 }} +{{- $script1 := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-versions-script" | trunc 52 }} +{{- if .Values.scripts.preprodRefresh.versions.enabled }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: preprod-refresh-versions-job +spec: + schedule: {{ .Values.scripts.preprodRefresh.versions.schedule }} + concurrencyPolicy: "Forbid" + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 345600 + backoffLimit: 0 + activeDeadlineSeconds: 7200 + template: + spec: + serviceAccountName: "book-a-secure-move-api" + containers: + - name: preprod-versions-refresh + image: "ghcr.io/ministryofjustice/hmpps-devops-tools:latest" + command: + - /bin/entrypoint.sh + volumeMounts: + - name: preprod-refresh-versions-script + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + env: + - name: DEST_DB_INSTANCE + valueFrom: + secretKeyRef: + name: preprod-rds-creds + key: url + - name: SOURCE_DB_INSTANCE + valueFrom: + secretKeyRef: + name: rds-instance-hmpps-book-secure-move-api-production + key: url + - name: AWS_DEFAULT_REGION + value: "eu-west-2" + restartPolicy: "Never" + volumes: + - name: preprod-refresh-versions-script + configMap: + name: {{ $script1 }} + defaultMode: 0755 +{{- end }} diff --git a/helm_deploy/hmpps-book-secure-move-api/values.yaml b/helm_deploy/hmpps-book-secure-move-api/values.yaml index 14c74a280..e75326928 100644 --- a/helm_deploy/hmpps-book-secure-move-api/values.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/values.yaml @@ -62,8 +62,14 @@ dashboards: scripts: preprodRefresh: - enabled: false - # schedule: "15 6 * * *" + main: + enabled: false + # schedule: "0 */4 * * *" + # dataSince: "6 hours ago" + versions: + enabled: false + # schedule: "15 7 * * *" + # dataSince: "26 hours ago" reports: enabled: false dailyIpt: diff --git a/helm_deploy/values-production.yaml b/helm_deploy/values-production.yaml index 97e39901a..04e95b6ee 100644 --- a/helm_deploy/values-production.yaml +++ b/helm_deploy/values-production.yaml @@ -119,8 +119,14 @@ dashboards: scripts: preprodRefresh: - enabled: true - schedule: "15 6 * * *" + main: + enabled: true + schedule: "0 */4 * * *" + dataSince: "6 hours ago" + versions: + enabled: true + schedule: "15 7 * * *" + dataSince: "26 hours ago" reports: enabled: true dailyIpt: