From 518b880d16d1a13b3787c3a191b7c23cce865e39 Mon Sep 17 00:00:00 2001 From: Daniel Benton Date: Fri, 12 Apr 2024 16:19:38 +0100 Subject: [PATCH 1/2] MAP-949 Preprod refresh scripts and cronjob --- .../01-create-temp-tables-sql.yaml | 45 ++ preprod-refresh/03-copy-temp-to-dest.yaml | 458 ++++++++++++++++++ preprod-refresh/04-drop-temp-tables.yaml | 45 ++ .../05-preprod-refresh-script.yaml | 116 +++++ preprod-refresh/06-refresh-cronjob.yaml | 71 +++ 5 files changed, 735 insertions(+) create mode 100644 preprod-refresh/01-create-temp-tables-sql.yaml create mode 100644 preprod-refresh/03-copy-temp-to-dest.yaml create mode 100644 preprod-refresh/04-drop-temp-tables.yaml create mode 100644 preprod-refresh/05-preprod-refresh-script.yaml create mode 100644 preprod-refresh/06-refresh-cronjob.yaml diff --git a/preprod-refresh/01-create-temp-tables-sql.yaml b/preprod-refresh/01-create-temp-tables-sql.yaml new file mode 100644 index 000000000..dbea6ae65 --- /dev/null +++ b/preprod-refresh/01-create-temp-tables-sql.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: create-temp-tables +data: + createTables.sql: |- + BEGIN; + CREATE TABLE framework_flags_temp (LIKE framework_flags); + CREATE TABLE framework_nomis_codes_temp (LIKE framework_nomis_codes); + CREATE TABLE framework_nomis_mappings_temp (LIKE framework_nomis_mappings); + CREATE TABLE framework_questions_temp (LIKE framework_questions); + CREATE TABLE framework_responses_temp (LIKE framework_responses); + CREATE TABLE frameworks_temp (LIKE frameworks); + CREATE TABLE framework_flags_responses_temp (LIKE framework_flags_responses); + CREATE TABLE framework_nomis_codes_questions_temp (LIKE framework_nomis_codes_questions); + CREATE TABLE framework_nomis_mappings_responses_temp (LIKE framework_nomis_mappings_responses); + CREATE TABLE locations_temp (LIKE locations); + CREATE TABLE moves_temp (LIKE moves); + CREATE TABLE journeys_temp (LIKE journeys); + CREATE TABLE people_temp (LIKE people); + CREATE TABLE profiles_temp (LIKE profiles); + CREATE TABLE person_escort_records_temp (LIKE person_escort_records); + CREATE TABLE court_hearings_temp (LIKE court_hearings); + CREATE TABLE allocation_complex_cases_temp (LIKE allocation_complex_cases); + CREATE TABLE allocations_temp (LIKE allocations); + CREATE TABLE assessment_questions_temp (LIKE assessment_questions); + CREATE TABLE categories_temp (LIKE categories); + CREATE TABLE documents_temp (LIKE documents); + CREATE TABLE ethnicities_temp (LIKE ethnicities); + CREATE TABLE flipper_features_temp (LIKE flipper_features); + CREATE TABLE flipper_gates_temp (LIKE flipper_gates); + CREATE TABLE genders_temp (LIKE genders); + CREATE TABLE locations_regions_temp (LIKE locations_regions); + CREATE TABLE nationalities_temp (LIKE nationalities); + CREATE TABLE nomis_alerts_temp (LIKE nomis_alerts); + CREATE TABLE notifications_temp (LIKE notifications); + CREATE TABLE populations_temp (LIKE populations); + CREATE TABLE regions_temp (LIKE regions); + CREATE TABLE subscriptions_temp (LIKE subscriptions); + CREATE TABLE supplier_locations_temp (LIKE supplier_locations); + CREATE TABLE suppliers_temp (LIKE suppliers); + CREATE TABLE youth_risk_assessments_temp (LIKE youth_risk_assessments); + CREATE TABLE generic_events_temp (LIKE generic_events); + -- CREATE TABLE versions_temp (LIKE versions); + COMMIT; diff --git a/preprod-refresh/03-copy-temp-to-dest.yaml b/preprod-refresh/03-copy-temp-to-dest.yaml new file mode 100644 index 000000000..7ddfb5b26 --- /dev/null +++ b/preprod-refresh/03-copy-temp-to-dest.yaml @@ -0,0 +1,458 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: copy-from-temp-sql +data: + copyFromTemp.sql: |- + BEGIN; + INSERT INTO frameworks SELECT * FROM frameworks_temp ON CONFLICT (id) DO UPDATE + SET + name = EXCLUDED.name, + version = EXCLUDED.version, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + + INSERT INTO framework_questions SELECT * FROM framework_questions_temp ON CONFLICT (id) DO UPDATE + SET + framework_id = EXCLUDED.framework_id, + key = EXCLUDED.key, + section = EXCLUDED.section, + required = EXCLUDED.required, + question_type = EXCLUDED.question_type, + options = EXCLUDED.options, + dependent_value = EXCLUDED.dependent_value, + followup_comment = EXCLUDED.followup_comment, + followup_comment_options = EXCLUDED.followup_comment_options, + parent_id = EXCLUDED.parent_id, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + prefill = EXCLUDED.prefill; + + INSERT INTO framework_flags SELECT * FROM framework_flags_temp ON CONFLICT (id) DO UPDATE + SET + framework_question_id = EXCLUDED.framework_question_id, + flag_type = EXCLUDED.flag_type, + title = EXCLUDED.title, + question_value = EXCLUDED.question_value, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + + + INSERT INTO framework_nomis_codes SELECT * FROM framework_nomis_codes_temp ON CONFLICT (id) DO UPDATE + SET + code_type = EXCLUDED.code_type, + code = EXCLUDED.code, + fallback = EXCLUDED.fallback, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + + INSERT INTO framework_nomis_mappings SELECT * FROM framework_nomis_mappings_temp ON CONFLICT (id) DO UPDATE + SET + raw_nomis_mapping = EXCLUDED.raw_nomis_mapping, + code = EXCLUDED.code, + code_type = EXCLUDED.code_type, + code_description = EXCLUDED.code_description, + comments = EXCLUDED.comments, + start_date = EXCLUDED.start_date, + end_date = EXCLUDED.end_date, + creation_date = EXCLUDED.creation_date, + expiry_date = EXCLUDED.expiry_date, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + approval_date = EXCLUDED.approval_date, + next_review_date = EXCLUDED.next_review_date; + + + INSERT INTO framework_responses SELECT * FROM framework_responses_temp ON CONFLICT (id) DO UPDATE + SET + framework_question_id = EXCLUDED.framework_question_id, + value_text = EXCLUDED.value_text, + value_json = EXCLUDED.value_json, + type = EXCLUDED.type, + parent_id = EXCLUDED.parent_id, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + responded = EXCLUDED.responded, + prefilled = EXCLUDED.prefilled, + assessmentable_id = EXCLUDED.assessmentable_id, + assessmentable_type = EXCLUDED.assessmentable_type, + value_type = EXCLUDED.value_type, + section = EXCLUDED.section, + responded_by = EXCLUDED.responded_by, + responded_at = EXCLUDED.responded_at; + + + + INSERT INTO framework_flags_responses_temp SELECT * FROM framework_flags_responses ON CONFLICT DO NOTHING; + TRUNCATE table framework_flags_responses; + INSERT INTO framework_flags_responses select DISTINCT framework_response_id, framework_flag_id from framework_flags_responses_temp; + + INSERT INTO framework_nomis_codes_questions_temp SELECT * FROM framework_nomis_codes_questions ON CONFLICT DO NOTHING; + TRUNCATE TABLE framework_nomis_codes_questions; + INSERT INTO framework_nomis_codes_questions SELECT DISTINCT framework_question_id, framework_nomis_code_id FROM framework_nomis_codes_questions_temp ON CONFLICT DO NOTHING; + + INSERT INTO framework_nomis_mappings_responses SELECT * FROM framework_nomis_mappings_responses_temp ON CONFLICT (id) DO UPDATE + SET + framework_response_id = EXCLUDED.framework_response_id, + framework_nomis_mapping_id = EXCLUDED.framework_nomis_mapping_id; + + + INSERT INTO categories SELECT * FROM categories_temp ON CONFLICT (id) DO UPDATE + SET + key = EXCLUDED.key, + title = EXCLUDED.title, + move_supported = EXCLUDED.move_supported, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + INSERT INTO ethnicities SELECT * FROM ethnicities_temp ON CONFLICT (id) DO UPDATE + SET + key = EXCLUDED.key, + title = EXCLUDED.title, + description = EXCLUDED.description, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + nomis_code = EXCLUDED.nomis_code, + disabled_at = EXCLUDED.disabled_at; + -- + + INSERT INTO genders SELECT * FROM genders_temp ON CONFLICT (id) DO UPDATE + SET + title = EXCLUDED.title, + description = EXCLUDED.description, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + key = EXCLUDED.key, + nomis_code = EXCLUDED.nomis_code, + disabled_at = EXCLUDED.disabled_at; + -- + + INSERT INTO locations SELECT * FROM locations_temp ON CONFLICT (id) DO UPDATE + SET + title = EXCLUDED.title, + location_type = EXCLUDED.location_type, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + nomis_agency_id = EXCLUDED.nomis_agency_id, + key = EXCLUDED.key, + disabled_at = EXCLUDED.disabled_at, + can_upload_documents = EXCLUDED.can_upload_documents, + category_id = EXCLUDED.category_id, + young_offender_institution = EXCLUDED.young_offender_institution, + premise = EXCLUDED.premise, + locality = EXCLUDED.locality, + city = EXCLUDED.city, + country = EXCLUDED.country, + postcode = EXCLUDED.postcode, + latitude = EXCLUDED.latitude, + longitude = EXCLUDED.longitude; + -- + INSERT INTO regions SELECT * FROM regions_temp ON CONFLICT (id) DO UPDATE + SET + name = EXCLUDED.name, + key = EXCLUDED.key, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + INSERT INTO suppliers SELECT * FROM suppliers_temp ON CONFLICT (id) DO UPDATE + SET + name = EXCLUDED.name, + key = EXCLUDED.key, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + INSERT INTO supplier_locations SELECT * FROM supplier_locations_temp ON CONFLICT (id) DO UPDATE + SET + supplier_id = EXCLUDED.supplier_id, + location_id = EXCLUDED.location_id, + effective_from = EXCLUDED.effective_from, + effective_to = EXCLUDED.effective_to, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + INSERT INTO locations_regions SELECT * FROM locations_regions_temp ON CONFLICT DO NOTHING; + -- + + INSERT INTO assessment_questions SELECT * FROM assessment_questions_temp ON CONFLICT (id) DO UPDATE + SET + title = EXCLUDED.title, + category = EXCLUDED.category, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + key = EXCLUDED.key, + disabled_at = EXCLUDED.disabled_at; + -- + + INSERT INTO nationalities SELECT * FROM nationalities_temp ON CONFLICT (id) DO UPDATE + SET + title = EXCLUDED.title, + description = EXCLUDED.description, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + key = EXCLUDED.key, + disabled_at = EXCLUDED.disabled_at; + -- + + INSERT INTO nomis_alerts SELECT * FROM nomis_alerts_temp ON CONFLICT (id) DO UPDATE + SET + type_code = EXCLUDED.type_code, + code = EXCLUDED.code, + description = EXCLUDED.description, + type_description = EXCLUDED.type_description, + assessment_question_id = EXCLUDED.assessment_question_id, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + INSERT INTO populations SELECT * FROM populations_temp ON CONFLICT (id) DO UPDATE + SET + location_id = EXCLUDED.location_id, + date = EXCLUDED.date, + operational_capacity = EXCLUDED.operational_capacity, + usable_capacity = EXCLUDED.usable_capacity, + unlock = EXCLUDED.unlock, + bedwatch = EXCLUDED.bedwatch, + overnights_in = EXCLUDED.overnights_in, + overnights_out = EXCLUDED.overnights_out, + out_of_area_courts = EXCLUDED.out_of_area_courts, + discharges = EXCLUDED.discharges, + updated_by = EXCLUDED.updated_by, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + INSERT INTO moves SELECT * FROM moves_temp ON CONFLICT (id) DO UPDATE + SET + date = EXCLUDED.date, + from_location_id = EXCLUDED.from_location_id, + to_location_id = EXCLUDED.to_location_id, + status = EXCLUDED.status, + updated_at = EXCLUDED.updated_at, + reference = EXCLUDED.reference, + move_type = EXCLUDED.move_type, + additional_information = EXCLUDED.additional_information, + nomis_event_id = EXCLUDED.nomis_event_id, + time_due = EXCLUDED.time_due, + cancellation_reason = EXCLUDED.cancellation_reason, + cancellation_reason_comment = EXCLUDED.cancellation_reason_comment, + profile_id = EXCLUDED.profile_id, + prison_transfer_reason_id = EXCLUDED.prison_transfer_reason_id, + reason_comment = EXCLUDED.reason_comment, + move_agreed = EXCLUDED.move_agreed, + move_agreed_by = EXCLUDED.move_agreed_by, + date_from = EXCLUDED.date_from, + date_to = EXCLUDED.date_to, + allocation_id = EXCLUDED.allocation_id, + rejection_reason = EXCLUDED.rejection_reason, + original_move_id = EXCLUDED.original_move_id, + supplier_id = EXCLUDED.supplier_id, + is_lockout = EXCLUDED.is_lockout; + -- + + INSERT INTO journeys SELECT * FROM journeys_temp ON CONFLICT (id) DO UPDATE + SET + move_id = EXCLUDED.move_id, + supplier_id = EXCLUDED.supplier_id, + from_location_id = EXCLUDED.from_location_id, + to_location_id = EXCLUDED.to_location_id, + billable = EXCLUDED.billable, + state = EXCLUDED.state, + vehicle = EXCLUDED.vehicle, + client_timestamp = EXCLUDED.client_timestamp, + updated_at = EXCLUDED.updated_at, + date = EXCLUDED.date; + -- + INSERT INTO people SELECT * FROM people_temp ON CONFLICT (id) DO UPDATE + SET + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + nomis_prison_number = EXCLUDED.nomis_prison_number, + prison_number = EXCLUDED.prison_number, + criminal_records_office = EXCLUDED.criminal_records_office, + police_national_computer = EXCLUDED.police_national_computer, + first_names = EXCLUDED.first_names, + last_name = EXCLUDED.last_name, + date_of_birth = EXCLUDED.date_of_birth, + gender_additional_information = EXCLUDED.gender_additional_information, + ethnicity_id = EXCLUDED.ethnicity_id, + gender_id = EXCLUDED.gender_id, + last_synced_with_nomis = EXCLUDED.last_synced_with_nomis, + latest_nomis_booking_id = EXCLUDED.latest_nomis_booking_id; + -- + + INSERT INTO profiles SELECT * FROM profiles_temp ON CONFLICT (id) DO UPDATE + SET + person_id = EXCLUDED.person_id, + last_name = EXCLUDED.last_name, + first_names = EXCLUDED.first_names, + date_of_birth = EXCLUDED.date_of_birth, + aliases = EXCLUDED.aliases, + gender_id = EXCLUDED.gender_id, + ethnicity_id = EXCLUDED.ethnicity_id, + nationality_id = EXCLUDED.nationality_id, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + assessment_answers = EXCLUDED.assessment_answers, + profile_identifiers = EXCLUDED.profile_identifiers, + gender_additional_information = EXCLUDED.gender_additional_information, + latest_nomis_booking_id = EXCLUDED.latest_nomis_booking_id, + category = EXCLUDED.category, + category_code = EXCLUDED.category_code, + category_id = EXCLUDED.category_id, + requires_youth_risk_assessment = EXCLUDED.requires_youth_risk_assessment, + csra = EXCLUDED.csra; + -- + commit; + begin; + INSERT INTO person_escort_records SELECT * FROM person_escort_records_temp ON CONFLICT (id) DO UPDATE + SET + framework_id = EXCLUDED.framework_id, + profile_id = EXCLUDED.profile_id, + status = EXCLUDED.status, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + confirmed_at = EXCLUDED.confirmed_at, + move_id = EXCLUDED.move_id, + nomis_sync_status = EXCLUDED.nomis_sync_status, + prefill_source_id = EXCLUDED.prefill_source_id, + completed_at = EXCLUDED.completed_at, + section_progress = EXCLUDED.section_progress, + amended_at = EXCLUDED.amended_at, + handover_details = EXCLUDED.handover_details, + handover_occurred_at = EXCLUDED.handover_occurred_at; + -- + + INSERT INTO allocation_complex_cases SELECT * FROM allocation_complex_cases_temp ON CONFLICT (id) DO UPDATE + SET + key = EXCLUDED.key, + title = EXCLUDED.title, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + + + INSERT INTO documents SELECT * FROM documents_temp ON CONFLICT (id) DO UPDATE + SET + move_id = EXCLUDED.move_id, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + discarded_at = EXCLUDED.discarded_at, + documentable_type = EXCLUDED.documentable_type, + documentable_id = EXCLUDED.documentable_id; + -- + + INSERT INTO flipper_features SELECT * FROM flipper_features_temp ON CONFLICT (id) DO UPDATE + SET + key = EXCLUDED.key, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + INSERT INTO flipper_gates SELECT * FROM flipper_gates_temp ON CONFLICT (id) DO UPDATE + SET + feature_key = EXCLUDED.feature_key, + key = EXCLUDED.key, + value = EXCLUDED.value, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + + INSERT INTO subscriptions SELECT * FROM subscriptions_temp ON CONFLICT DO NOTHING; + -- + + INSERT INTO notifications SELECT * FROM notifications_temp ON CONFLICT (id) DO UPDATE + SET + subscription_id = EXCLUDED.subscription_id, + event_type = EXCLUDED.event_type, + topic_id = EXCLUDED.topic_id, + topic_type = EXCLUDED.topic_type, + delivery_attempts = EXCLUDED.delivery_attempts, + delivery_attempted_at = EXCLUDED.delivery_attempted_at, + delivered_at = EXCLUDED.delivered_at, + discarded_at = EXCLUDED.discarded_at, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + response_id = EXCLUDED.response_id, + notification_type_id = EXCLUDED.notification_type_id; + -- + + + INSERT INTO allocations SELECT * FROM allocations_temp ON CONFLICT (id) DO UPDATE + SET + from_location_id = EXCLUDED.from_location_id, + to_location_id = EXCLUDED.to_location_id, + date = EXCLUDED.date, + prisoner_category = EXCLUDED.prisoner_category, + sentence_length = EXCLUDED.sentence_length, + complex_cases = EXCLUDED.complex_cases, + moves_count = EXCLUDED.moves_count, + complete_in_full = EXCLUDED.complete_in_full, + other_criteria = EXCLUDED.other_criteria, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + status = EXCLUDED.status, + cancellation_reason = EXCLUDED.cancellation_reason, + cancellation_reason_comment = EXCLUDED.cancellation_reason_comment, + requested_by = EXCLUDED.requested_by, + estate = EXCLUDED.estate, + sentence_length_comment = EXCLUDED.sentence_length_comment, + estate_comment = EXCLUDED.estate_comment; + -- + + + + INSERT INTO court_hearings SELECT * FROM court_hearings_temp ON CONFLICT (id) DO UPDATE + SET + move_id = EXCLUDED.move_id, + start_time = EXCLUDED.start_time, + case_start_date = EXCLUDED.case_start_date, + case_type = EXCLUDED.case_type, + comments = EXCLUDED.comments, + case_number = EXCLUDED.case_number, + nomis_case_id = EXCLUDED.nomis_case_id, + nomis_hearing_id = EXCLUDED.nomis_hearing_id, + saved_to_nomis = EXCLUDED.saved_to_nomis, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at; + -- + + INSERT INTO youth_risk_assessments SELECT * FROM youth_risk_assessments_temp ON CONFLICT (id) DO UPDATE + SET + framework_id = youth_risk_assessments.framework_id, + profile_id = EXCLUDED.profile_id, + move_id = EXCLUDED.move_id, + prefill_source_id = EXCLUDED.prefill_source_id, + status = EXCLUDED.status, + nomis_sync_status = EXCLUDED.nomis_sync_status, + confirmed_at = EXCLUDED.confirmed_at, + completed_at = EXCLUDED.completed_at, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + section_progress = EXCLUDED.section_progress; + -- + COMMIT; + -- + BEGIN; + INSERT INTO generic_events SELECT * FROM generic_events_temp ON CONFLICT (id) DO UPDATE + SET + eventable_id = EXCLUDED.eventable_id, + eventable_type = EXCLUDED.eventable_type, + type = EXCLUDED.type, + notes = EXCLUDED.notes, + created_by = EXCLUDED.created_by, + details = EXCLUDED.details, + occurred_at = EXCLUDED.occurred_at, + recorded_at = EXCLUDED.recorded_at, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + supplier_id = EXCLUDED.supplier_id, + classification = EXCLUDED.classification; + -- + + -- INSERT INTO versions (id, item_type, item_id, event, whodunnit, object, created_at, supplier_id) SELECT nextval('versions_id_seq'::regclass), item_type, item_id, event, whodunnit, object, created_at, supplier_id FROM versions_temp; + COMMIT; + diff --git a/preprod-refresh/04-drop-temp-tables.yaml b/preprod-refresh/04-drop-temp-tables.yaml new file mode 100644 index 000000000..92a50bcc0 --- /dev/null +++ b/preprod-refresh/04-drop-temp-tables.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: drop-temp-tables +data: + dropTables.sql: |- + BEGIN; + DROP TABLE IF EXISTS framework_flags_temp CASCADE; + DROP TABLE IF EXISTS framework_nomis_codes_temp CASCADE; + DROP TABLE IF EXISTS framework_nomis_mappings_temp CASCADE; + DROP TABLE IF EXISTS framework_questions_temp CASCADE; + DROP TABLE IF EXISTS framework_responses_temp CASCADE; + DROP TABLE IF EXISTS frameworks_temp CASCADE; + DROP TABLE IF EXISTS framework_flags_responses_temp CASCADE; + DROP TABLE IF EXISTS framework_nomis_codes_questions_temp CASCADE; + DROP TABLE IF EXISTS framework_nomis_mappings_responses_temp CASCADE; + DROP TABLE IF EXISTS locations_temp CASCADE; + DROP TABLE IF EXISTS moves_temp CASCADE; + DROP TABLE IF EXISTS journeys_temp CASCADE; + DROP TABLE IF EXISTS people_temp CASCADE; + DROP TABLE IF EXISTS profiles_temp CASCADE; + DROP TABLE IF EXISTS person_escort_records_temp CASCADE; + DROP TABLE IF EXISTS court_hearings_temp CASCADE; + DROP TABLE IF EXISTS allocation_complex_cases_temp CASCADE; + DROP TABLE IF EXISTS allocations_temp CASCADE; + DROP TABLE IF EXISTS assessment_questions_temp CASCADE; + DROP TABLE IF EXISTS categories_temp CASCADE; + DROP TABLE IF EXISTS documents_temp CASCADE; + DROP TABLE IF EXISTS ethnicities_temp CASCADE; + DROP TABLE IF EXISTS flipper_features_temp CASCADE; + DROP TABLE IF EXISTS flipper_gates_temp CASCADE; + DROP TABLE IF EXISTS genders_temp CASCADE; + DROP TABLE IF EXISTS locations_regions_temp CASCADE; + DROP TABLE IF EXISTS nationalities_temp CASCADE; + DROP TABLE IF EXISTS nomis_alerts_temp CASCADE; + DROP TABLE IF EXISTS notifications_temp CASCADE; + DROP TABLE IF EXISTS populations_temp CASCADE; + DROP TABLE IF EXISTS regions_temp CASCADE; + DROP TABLE IF EXISTS subscriptions_temp CASCADE; + DROP TABLE IF EXISTS supplier_locations_temp CASCADE; + DROP TABLE IF EXISTS suppliers_temp CASCADE; + DROP TABLE IF EXISTS youth_risk_assessments_temp CASCADE; + DROP TABLE IF EXISTS generic_events_temp CASCADE; + DROP TABLE IF EXISTS versions_temp CASCADE; + COMMIT; diff --git a/preprod-refresh/05-preprod-refresh-script.yaml b/preprod-refresh/05-preprod-refresh-script.yaml new file mode 100644 index 000000000..804b7d2e6 --- /dev/null +++ b/preprod-refresh/05-preprod-refresh-script.yaml @@ -0,0 +1,116 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: preprod-refresh-script +data: + entrypoint.sh: |- + #!/bin/bash + from=$(date --date="5 days ago" '+%Y-%m-%d') + to=$(date '+%Y-%m-%d') + echo "Refreshing data from ${from} to ${to}" + + createTemp='/tmp/sql/createTables.sql' + copyScript='/tmp/sql/copyFromTemp.sql' + dropScript='/tmp/sql/dropTables.sql' + + execDest() { + cmd=$1 + psql $DEST_DB_INSTANCE -c "${cmd}" + } + + execSource() { + cmd=$1 + psql $SOURCE_DB_INSTANCE -c "${cmd}" + } + + execFileDest() { + file=$1 + psql $DEST_DB_INSTANCE -f ${file} + } + + execFileSource() { + file=$1 + psql $SOURCE_DB_INSTANCE -f ${file} + } + + temp () { + table=$1 + echo "Copying ${table} to source temp table" + cmd="INSERT INTO ${table}_temp SELECT * FROM ${table} where (created_at >= '${from}' and created_at < '${to}') OR (updated_at >= '${from}' and updated_at < '${to}');" + execSource "${cmd}" + } + + tempjoin () { + table=$1 + join=$2 + echo "Copying ${table} to temporary table - using explicit select statement" + cmd_two="INSERT INTO ${table}_temp ${join};" + execSource "${cmd_two}" + } + + execFileSource ${dropScript} + execFileDest ${dropScript} + execFileDest ${createTemp} + execFileSource ${createTemp} + + temp framework_flags + temp framework_nomis_codes + temp framework_nomis_mappings + temp framework_questions + temp framework_responses + temp frameworks + + #JOIN TABLES + tempjoin framework_flags_responses "SELECT framework_response_id, framework_flag_id FROM public.framework_flags_responses JOIN framework_responses_temp ON framework_responses_temp.id = framework_flags_responses.framework_response_id" + tempjoin framework_nomis_codes_questions "SELECT framework_question_id, framework_nomis_code_id FROM framework_nomis_codes_questions JOIN framework_questions_temp ON framework_questions_temp.id = framework_nomis_codes_questions.framework_question_id" + tempjoin framework_nomis_mappings_responses "SELECT framework_nomis_mappings_responses.id, framework_response_id, framework_nomis_mapping_id FROM framework_nomis_mappings_responses JOIN framework_responses_temp ON framework_responses_temp.id = framework_nomis_mappings_responses.framework_response_id" + + temp locations + temp moves + temp journeys + temp people + temp profiles + temp person_escort_records + temp court_hearings + + #tempall active_storage_attachments #Purposefully omitted + #tempall active_storage_blobs #Purposefully omitted + #tempall active_storage_variant_records #Purposefully omitted + temp allocation_complex_cases + temp allocations + #temp ar_internal_metadata #Purposefully omitted + temp assessment_questions + temp categories + temp documents + temp ethnicities + temp flipper_features + temp flipper_gates + + temp genders + + temp locations_regions + temp nationalities + temp nomis_alerts + temp notifications + #temp oauth_access_grants #Purposefully omitted + #temp oauth_access_tokens #Purposefully omitted + #temp oauth_applications #Purposefully omitted + temp populations + temp regions + #temp schema_migrations #Purposefully omitted + temp subscriptions + temp supplier_locations + temp suppliers + temp youth_risk_assessments + temp generic_events + + echo "Starting dump of data to preprod" + pg_dump $SOURCE_DB_INSTANCE -t "*_temp" -O --section=data | psql $DEST_DB_INSTANCE + echo "Finished dumping data" + + echo "Copying data from destination temp tables to actual tables" + execFileDest ${copyScript} + echo "Dropping temporary tables" + execFileSource ${dropScript} + execFileDest ${dropScript} + echo "All done!" \ No newline at end of file diff --git a/preprod-refresh/06-refresh-cronjob.yaml b/preprod-refresh/06-refresh-cronjob.yaml new file mode 100644 index 000000000..0778f8da0 --- /dev/null +++ b/preprod-refresh/06-refresh-cronjob.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: preprod-refresh-job +spec: + schedule: "0 4 1 * *" + concurrencyPolicy: "Forbid" + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 345600 + backoffLimit: 0 + activeDeadlineSeconds: 1200 + template: + spec: + serviceAccountName: "book-a-secure-move-api" + containers: + - name: map-949 + image: "ghcr.io/ministryofjustice/hmpps-devops-tools:latest" + command: + - /bin/entrypoint.sh + volumeMounts: + - name: create-temp-tables + mountPath: /tmp/sql/createTables.sql + readOnly: true + subPath: createTables.sql + - name: copy-from-temp-sql + mountPath: /tmp/sql/copyFromTemp.sql + readOnly: true + subPath: copyFromTemp.sql + - name: drop-temp-tables + mountPath: /tmp/sql/dropTables.sql + readOnly: true + subPath: dropTables.sql + - name: preprod-refresh-script + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + env: + - name: DEST_DB_INSTANCE + valueFrom: + secretKeyRef: + name: preprod-rds-creds + key: url + - name: SOURCE_DB_INSTANCE + valueFrom: + secretKeyRef: + name: rds-instance-hmpps-book-secure-move-api-production + key: url + - name: AWS_DEFAULT_REGION + value: "eu-west-2" + restartPolicy: "Never" + volumes: + - name: create-temp-tables + configMap: + name: create-temp-tables + defaultMode: 0755 + - name: copy-from-temp-sql + configMap: + name: copy-from-temp-sql + defaultMode: 0755 + - name: drop-temp-tables + configMap: + name: drop-temp-tables + defaultMode: 0755 + - name: preprod-refresh-script + configMap: + name: preprod-refresh-script + defaultMode: 0755 \ No newline at end of file From ece2890914e5d63e74be0f345ae37500cec7e41a Mon Sep 17 00:00:00 2001 From: Daniel Benton Date: Wed, 17 Apr 2024 08:11:36 +0100 Subject: [PATCH 2/2] MAP-949 Update preprod refresh files to match what's running --- preprod-refresh/05-preprod-refresh-script.yaml | 2 +- preprod-refresh/06-refresh-cronjob.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/preprod-refresh/05-preprod-refresh-script.yaml b/preprod-refresh/05-preprod-refresh-script.yaml index 804b7d2e6..9af6d5b82 100644 --- a/preprod-refresh/05-preprod-refresh-script.yaml +++ b/preprod-refresh/05-preprod-refresh-script.yaml @@ -5,7 +5,7 @@ metadata: data: entrypoint.sh: |- #!/bin/bash - from=$(date --date="5 days ago" '+%Y-%m-%d') + from=$(date --date="2 days ago" '+%Y-%m-%d') to=$(date '+%Y-%m-%d') echo "Refreshing data from ${from} to ${to}" diff --git a/preprod-refresh/06-refresh-cronjob.yaml b/preprod-refresh/06-refresh-cronjob.yaml index 0778f8da0..d33240b58 100644 --- a/preprod-refresh/06-refresh-cronjob.yaml +++ b/preprod-refresh/06-refresh-cronjob.yaml @@ -4,7 +4,7 @@ kind: CronJob metadata: name: preprod-refresh-job spec: - schedule: "0 4 1 * *" + schedule: "15 6 * * *" concurrencyPolicy: "Forbid" successfulJobsHistoryLimit: 5 failedJobsHistoryLimit: 3 @@ -12,12 +12,12 @@ spec: spec: ttlSecondsAfterFinished: 345600 backoffLimit: 0 - activeDeadlineSeconds: 1200 + activeDeadlineSeconds: 7200 template: spec: serviceAccountName: "book-a-secure-move-api" containers: - - name: map-949 + - name: preprod-refresh image: "ghcr.io/ministryofjustice/hmpps-devops-tools:latest" command: - /bin/entrypoint.sh