Modify backup / restore roles according to test results (#1356)

* merge, modify postgres to use new setup * recovery fix * centos testing changes * fix after redhat tests * merge * Add provider to backup / restore * Notes about kubernetes
hitachienergy · Jun 16, 2020 · dc3be13 · dc3be13
1 parent d9b5713
commit dc3be13
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 43 deletions.
diff --git a/...src/epicli/data/common/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml b/...src/epicli/data/common/ansible/playbooks/roles/backup/tasks/common/download_via_rsync.yml
@@ -36,6 +36,11 @@
               {{ openssh_keypair.public_key }}
 
   block:
+    - name: Ensure that .ssh directory exists
+      file:
+        path: ~/.ssh/
+        state: directory
+
     - name: Create a temporary file path to hold the private key in
       tempfile:
         path: ~/.ssh/

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/common/download_via_ssh.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/common/download_via_ssh.yml
@@ -39,6 +39,11 @@
               {{ openssh_keypair.public_key }}
 
   block:
+    - name: Ensure that .ssh directory exists
+      file:
+        path: ~/.ssh/
+        state: directory
+
     - name: Create a temporary file path to hold the private key in
       tempfile:
         path: ~/.ssh/

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml
@@ -21,7 +21,6 @@
 
 - name: Create and export backup
   block:
-
   - name: Create temporary directories for backup files
     file:
       path: "/var/tmp/{{ snapshot_name }}/{{ item }}"
@@ -42,7 +41,7 @@
       chdir: "{{ pg_config_dir[ansible_os_family] }}"
     register: config_files
 
-  - name: Copy config files into teporary location
+  - name: Copy config files into temporary location
     copy:
       src: "{{ pg_config_dir[ansible_os_family] }}/{{ item }}"
       dest: "/var/tmp/{{ snapshot_name  }}/configs"
@@ -70,6 +69,5 @@
     file:
       path: "/var/tmp/{{ snapshot_name  }}/"
       state: absent
-
   when: (groups['postgresql'][0] == inventory_hostname and hostvars[groups['postgresql'][0]]['node0'].rc == 0) or
         (groups['postgresql'][1] == inventory_hostname and hostvars[groups['postgresql'][0]]['node0'].rc != 0)
diff --git a/...epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_definitions.yml b/...epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_definitions.yml
@@ -28,7 +28,7 @@
 
 - name: Save definitions in a json file
   shell: |
-    rabbitmqadmin export /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json
+    /usr/local/bin/rabbitmqadmin export /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json
   args:
     executable: /bin/bash
 

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml
@@ -1,29 +1,28 @@
 ---
+# Actions taken only when replication with repmgr is enabled
 - name: Stop repmgr service
-
   block:
-  - name: Stop repmgr on standby server
+  - name: Stop repmgr on standby node
     service:
       name: "{{ repmgr_service_name[ansible_os_family] }}"
       state: stopped
     when:
       - groups['postgresql'][1] == inventory_hostname
 
-  - name: Stop repmgr on primary server
+  - name: Stop repmgr on primary node
     service:
       name: "{{ repmgr_service_name[ansible_os_family] }}"
       state: stopped
     when:
       - groups['postgresql'][0] == inventory_hostname
-
   when: 
     - component_vars.specification.extensions.replication.enabled is defined
     - component_vars.specification.extensions.replication.enabled
     - component_vars.specification.extensions.replication.use_repmgr is defined
     - component_vars.specification.extensions.replication.use_repmgr
 
 - name: Copy and restore backup files
-
+  # Running on primary or only node
   block:
     - name: Find snapshot archive
       import_tasks: common/find_snapshot_archive.yml
@@ -56,7 +55,7 @@
         src: "{{ recovery_dir }}/{{ snapshot_path | basename }}"
         remote_src: true
 
-    - name: Save existing configuration files
+    - name: Cache existing configuration files
       block:
         - name: List existing configuration files
           shell: "find *.conf"
@@ -80,9 +79,18 @@
       file:
         path: "{{ pg_data_dir[ansible_os_family] }}/"
         state: absent
-
-    - name: Initialize database (Red Hat)
-      command: "/usr/pgsql-10/bin/postgresql-10-setup initdb {{ pg_service_name[ansible_os_family] }}"
+
+    - name: Initialize database (RedHat)
+      block:
+      - name: Ensure that data directory exists
+        file:
+          path: "{{ pg_data_dir[ansible_os_family] }}"
+          state: directory
+          owner: postgres
+          group: postgres
+
+      - name: Initialize database
+        command: "/usr/pgsql-10/bin/postgresql-10-setup initdb {{ pg_service_name[ansible_os_family] }}"
       when:
         - ansible_os_family == 'RedHat'
 
@@ -93,7 +101,7 @@
       when:
         - ansible_os_family == 'Debian'
 
-    - name: Copy config files from temporary location
+    - name: Copy cached config files
       copy:
         src: "/var/tmp/postgresql_temp_config/"
         dest: "{{ pg_config_dir[ansible_os_family] }}/"
@@ -106,30 +114,31 @@
         name: "{{ pg_service_name[ansible_os_family] }}"
         state: started
 
-    - name: Import database dump file
+    - name: Import database from dump file
       become: yes
       become_user: postgres
       command: "psql -f /var/tmp/postgresql_restore_source/data/database_dump.sql postgres"
-
-    - name: Tasks related to repmgr
+
+    - name: Configure repmgr
+      #Repmgr on primary node
       block:
-      - name: Start repmgr on primary server
-        service:
-          name: "{{ repmgr_service_name[ansible_os_family] }}"
-          state: started
-
       - name: Register primary node in repmgr
         become: yes
         become_user: postgres
         shell: "{{ repmgr_bindir[ansible_os_family] }}/repmgr -f {{ repmgr_config_dir[ansible_os_family] }}/repmgr.conf
         --force --superuser={{ component_vars.specification.extensions.replication.priviledged_user_name }} primary register -F"
+
+      - name: Start repmgr on primary node
+        service:
+          name: "{{ repmgr_service_name[ansible_os_family] }}"
+          state: started
       when:
         - component_vars.specification.extensions.replication.enabled is defined
         - component_vars.specification.extensions.replication.enabled
         - component_vars.specification.extensions.replication.use_repmgr is defined
         - component_vars.specification.extensions.replication.use_repmgr
 
-    - name: Remove temporary files
+    - name: Remove created temporary files
       file:
         path: "{{ item }}"
         state: absent
@@ -141,7 +150,7 @@
   when:
     - groups['postgresql'][0] == inventory_hostname
 
-- name: tasks related to repmgr configuration on secondary node
+- name: Configure repmgr on secondary node
   block:
     - name: Stop postgresql service
       service:
@@ -154,9 +163,9 @@
         state: directory
         mode: 0666
 
-    - name: Save existing configuration files
+    - name: Cache existing configuration files
       block:
-        - name: search for existing configuration files
+        - name: search for existing configuration files (RedHat)
           shell: "find *.conf"
           args:
             chdir: "{{ pg_config_dir[ansible_os_family] }}"
@@ -169,28 +178,32 @@
             remote_src: yes
           loop: "{{ config_files.stdout_lines|flatten(levels=1) }}"
 
-    - name: Clean existing data directory before clonning from primary node
+    - name: Delete existing data directory before cloning from primary node
       file:
         path: "{{ pg_data_dir[ansible_os_family] }}/"
         state: absent
 
-    - name: Copy config files back to database configuration location
-      copy:
-        src: "/var/tmp/postgresql_temp_config/"
-        dest: "{{ pg_config_dir[ansible_os_family] }}/"
+    - name: Ensure that data directory exists (RedHat) #This needs to be check pn RedHat family sytems since not always location is created by init
+      file:
+        path: "{{ pg_data_dir[ansible_os_family] }}/"
+        state: directory
         owner: postgres
         group: postgres
-        remote_src: yes
-
-    - name: Start repmgr service
-      service:
-        name: "{{ repmgr_service_name[ansible_os_family] }}"
-        state: started
+      when: 
+        - ansible_os_family == 'RedHat'
 
     - name: Clone content from primary node using repmgr
       become_user: postgres
       shell: "{{ repmgr_bindir[ansible_os_family] }}/repmgr -f {{ repmgr_config_dir[ansible_os_family] }}/repmgr.conf -h {{ hostvars[groups['postgresql'][0]]['ansible_default_ipv4']['address'] }} -U {{ component_vars.specification.extensions.replication.priviledged_user_name }} -d {{ component_vars.specification.extensions.replication.repmgr_database }} -p 5432 -F standby clone"
 
+    - name: Copy cached config files back to database configuration location
+      copy:
+        src: "/var/tmp/postgresql_temp_config/"
+        dest: "{{ pg_config_dir[ansible_os_family] }}/"
+        owner: postgres
+        group: postgres
+        remote_src: yes
+
     - name: Start postgresql service
       service:
         name: "{{ pg_service_name[ansible_os_family] }}"
@@ -200,6 +213,11 @@
       become_user: postgres
       shell: "{{ repmgr_bindir[ansible_os_family] }}/repmgr -f {{ repmgr_config_dir[ansible_os_family] }}/repmgr.conf standby register -F"
 
+    - name: Start repmgr service
+      service:
+        name: "{{ repmgr_service_name[ansible_os_family] }}"
+        state: started
+
     - name: Rejoin secondary node to repmgr cluster
       become_user: postgres
       shell: "{{ repmgr_bindir[ansible_os_family] }}/repmgr -f {{ repmgr_config_dir[ansible_os_family] }}/repmgr.conf standby follow -F"

diff --git a/...icli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_definitions.yml b/...icli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_definitions.yml
@@ -50,6 +50,6 @@
 
 - name: Import definitions json file
   shell: |
-    rabbitmqadmin import /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json
+    /usr/local/bin/rabbitmqadmin import /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json
   args:
     executable: /bin/bash
diff --git a/core/src/epicli/data/common/defaults/configuration/backup.yml b/core/src/epicli/data/common/defaults/configuration/backup.yml
@@ -3,8 +3,6 @@ title: Backup Config
 name: default
 specification:
   components:
-    kubernetes:
-      enabled: false
     load_balancer:
       enabled: false
     logging:
@@ -15,3 +13,7 @@ specification:
       enabled: false
     rabbitmq:
       enabled: false
+#Kubernes backup and recovery is not supported by Epiphany. 
+#You may create backup by enabling this below, but recovery should be done manually according Kubernetes documentation.
+    kubernetes:
+      enabled: false
diff --git a/core/src/epicli/data/common/defaults/configuration/recovery.yml b/core/src/epicli/data/common/defaults/configuration/recovery.yml
@@ -3,9 +3,6 @@ title: Recovery Config
 name: default
 specification:
   components:
-    kubernetes:
-      enabled: false
-      snapshot_name: latest
     load_balancer:
       enabled: false
       snapshot_name: latest
@@ -21,3 +18,8 @@ specification:
     rabbitmq:
       enabled: false
       snapshot_name: latest
+#This is a dangerous procedure that is currently disabled because it needs refactoring!
+#Kubernetes recovery should be done manually according Kubernetes documentation
+#    kubernetes:
+#      enabled: false
+#      snapshot_name: latest