backup/recovery: uncommenting and refactoring legacy code (#1241)

core/src/epicli/cli/engine/PatchEngine.py

@@ -11,6 +11,7 @@ class PatchEngine(Step):
     def __init__(self, input_data):
         super().__init__(__name__)
         self.build_directory = input_data.build_directory
+        self.components = input_data.components
         self.ansible_command = AnsibleCommand()
 
     def __enter__(self):
@@ -21,26 +22,28 @@ def __exit__(self, exc_type, exc_value, traceback):
         super().__exit__(exc_type, exc_value, traceback)
 
     def backup(self):
-        self.upgrade_patch_files_and_run('backup')
+        for component in sorted(self.components):
+            self.upgrade_patch_files_and_run('backup', component)
         return 0
 
     def recovery(self):
-        self.upgrade_patch_files_and_run('recovery')
+        for component in sorted(self.components):
+            self.upgrade_patch_files_and_run('recovery', component)
         return 0
 
-    def upgrade_patch_files_and_run(self, action):
-        self.logger.info(f'Running {action}...')
+    def upgrade_patch_files_and_run(self, action, component):
+        self.logger.info(f'Running {action} on {component}...')
 
         #copy role files
         roles_build_path = os.path.join(self.build_directory, 'ansible/roles', action)
         roles_source_path = os.path.join(AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH, 'roles', action)
         copy_files_recursively(roles_source_path, roles_build_path)
 
         #copy playbook file
-        playbook_build_path = os.path.join(self.build_directory, 'ansible/') + action + '.yml'
-        playbook_source_path = os.path.join(AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH) + action + '.yml' 
-        copy_file(playbook_source_path, playbook_build_path)  
-        
+        playbook_build_path = os.path.join(self.build_directory, 'ansible/') + action + '_' + component + '.yml'
+        playbook_source_path = os.path.join(AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH) + action + '_' + component + '.yml'
+        copy_file(playbook_source_path, playbook_build_path)
+
         #run the playbook
         inventory_path = get_inventory_path_for_build(self.build_directory)
-        self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=playbook_build_path)        
+        self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=playbook_build_path)

core/src/epicli/cli/epicli.py

@@ -91,12 +91,11 @@ def debug_level(x):
     upgrade_parser(subparsers)
     delete_parser(subparsers)
     test_parser(subparsers)
-
     '''
     validate_parser(subparsers)
+    '''
     backup_parser(subparsers)
     recovery_parser(subparsers)
-    '''
 
     # check if there were any variables and display full help
     if len(sys.argv) < 2:
@@ -253,6 +252,24 @@ def run_validate(args):
             return engine.validate()
 
     sub_parser.set_defaults(func=run_validate)    
+'''
+
+
+def _component_parser_for(available_components={}):
+    def parse_components(value):
+        parsed_items = set(
+            item_stripped
+            for item in value.split(',')
+            for item_stripped in [item.strip()]
+            if item_stripped
+        )
+        if len(parsed_items) == 1 and 'all' in parsed_items:
+            return set(available_components)
+        difference = parsed_items - set(available_components)
+        if difference:
+            raise Exception('Error parsing components: invalid values present')
+        return parsed_items
+    return parse_components
 
 
 def backup_parser(subparsers):
@@ -261,6 +278,15 @@ def backup_parser(subparsers):
     sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
                             help='Absolute path to directory with build artifacts.')
 
+    available_components = {'kubernetes', 'postgresql', 'monitoring'}
+
+    enabled_components = set(available_components)  # enable everything by default
+    enabled_components_joined = ','.join(sorted(enabled_components))
+
+    sub_parser.add_argument('-c', '--components', dest='components', type=_component_parser_for(available_components), required=False,
+                            help=f'Specify comma-separated list of components to backup (defaults to "{enabled_components_joined}").',
+                            default=enabled_components_joined)
+
     def run_backup(args):
         experimental_query()
         adjust_paths_from_build(args)
@@ -271,18 +297,27 @@ def run_backup(args):
 
 
 def recovery_parser(subparsers):
-    sub_parser = subparsers.add_parser('recovery', description='[Experimental]: Recover from existing backup.')
+    sub_parser = subparsers.add_parser('recovery',
+                                       description='[Experimental]: Recover from existing backup.')
     sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
                             help='Absolute path to directory with build artifacts.')
 
+    available_components = {'kubernetes', 'postgresql', 'monitoring'}
+
+    enabled_components = set()  # disable everything by default
+    enabled_components_joined = ','.join(sorted(enabled_components))
+
+    sub_parser.add_argument('-c', '--components', dest='components', type=_component_parser_for(available_components), required=False,
+                            help=f'Specify comma-separated list of components to restore (defaults to "{enabled_components_joined}").',
+                            default=enabled_components_joined)
+
     def run_recovery(args):
         experimental_query()
         adjust_paths_from_build(args)
         with PatchEngine(args) as engine:
             return engine.recovery()
 
     sub_parser.set_defaults(func=run_recovery)
-'''
 
 
 def experimental_query():

core/src/epicli/data/common/ansible/playbooks/backup_kubernetes.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for backing up Kubernetes cluster
+
+- hosts: kubernetes_master[0]
+  become: true
+  become_method: sudo
+  serial: 1
+  tasks:
+    - import_role:
+        name: backup
+        tasks_from: kubernetes

core/src/epicli/data/common/ansible/playbooks/backup_monitoring.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for backing up monitoring data
+
+- hosts: prometheus
+  become: true
+  become_method: sudo
+  serial: 1
+  tasks:
+    - import_role:
+        name: backup
+        tasks_from: monitoring

core/src/epicli/data/common/ansible/playbooks/backup_postgresql.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for backing up Postgresql database
+
+- hosts: postgresql[0]
+  become: true
+  become_method: sudo
+  serial: 1
+  tasks:
+    - import_role:
+        name: backup
+        tasks_from: postgresql

core/src/epicli/data/common/ansible/playbooks/recovery_kubernetes.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for recovering Kubernetes cluster
+
+- hosts: kubernetes_master[0]
+  become: true
+  become_method: sudo
+  serial: 1
+  tasks:
+    - import_role:
+        name: recovery
+        tasks_from: kubernetes

core/src/epicli/data/common/ansible/playbooks/recovery_monitoring.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for recovering monitoring data
+
+- hosts: prometheus
+  become: true
+  become_method: sudo
+  serial: 1
+  tasks:
+    - import_role:
+        name: recovery
+        tasks_from: monitoring

core/src/epicli/data/common/ansible/playbooks/recovery_postgresql.yml

@@ -0,0 +1,11 @@
+---
+# Ansible playbook for recovering Postgresql database
+
+- hosts: postgresql[0]
+  become: true
+  become_method: sudo
+  serial: 1
+  tasks:
+    - import_role:
+        name: recovery
+        tasks_from: postgresql

core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-backup_dir: /home/{{ admin_user.name }}/backupdir
+backup_dir: /epibackup

core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/kubernetes.yml

@@ -0,0 +1,104 @@
+---
+- name: Ensure backup directory exists
+  file:
+    path: "{{ backup_dir }}/"
+    state: directory
+    mode: u=rwx,go=
+
+- name: Create temporary directory
+  tempfile:
+    path: "{{ backup_dir }}/"
+    suffix: .tmp
+    state: directory
+  register: backup_temp_dir
+
+- name: Save backup and cleanup afterwards
+  always:
+    - name: Delete temporary directory
+      file:
+        path: "{{ backup_temp_dir.path }}/"
+        state: absent
+  block:
+    - name: Get etcd image name
+      shell: |
+        kubectl get pods \
+          --all-namespaces \
+          --output jsonpath={{ jsonpath }}
+      vars:
+        jsonpath: >-
+          "{.items[*].spec.containers[?(@.name=='etcd')].image}"
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      register: etcd_image_name
+
+    - name: Save etcd image name to a file
+      copy:
+        dest: "{{ backup_temp_dir.path }}/etcd_ver.txt"
+        content: |-
+          {{ etcd_image_name.stdout | trim }}
+
+    - name: Save kubernetes PKI
+      copy:
+        src: /etc/kubernetes/pki  # do not put / at the end here!
+        dest: "{{ backup_temp_dir.path }}/"
+        remote_src: true
+
+    - name: Save etcd snapshot
+      shell: |
+        docker run \
+        -v "{{ backup_temp_dir.path }}/:/backup/" \
+        --network host \
+        --env ETCDCTL_API=3 \
+        --rm "{{ etcd_image_name.stdout | trim }}" \
+        etcdctl \
+        --endpoints https://127.0.0.1:2379 \
+        --cacert /backup/pki/etcd/ca.crt \
+        --cert /backup/pki/etcd/healthcheck-client.crt \
+        --key /backup/pki/etcd/healthcheck-client.key \
+        snapshot save /backup/etcd-snapshot.db
+      args:
+        executable: /bin/bash
+
+    - name: Check if kubeadm configuration file exists
+      stat:
+        path: /etc/kubeadm/kubeadm-config.yml
+        get_attributes: false
+        get_checksum: false
+        get_mime: false
+      register: stat_kubeadm_config_yml
+
+    - when: stat_kubeadm_config_yml.stat.exists
+      block:
+        - name: Save kubeadm configuration file
+          copy:
+            src: "{{ stat_kubeadm_config_yml.stat.path }}"
+            dest: "{{ backup_temp_dir.path }}/"
+            remote_src: true
+
+    - name: Get current timestamp
+      run_once: true
+      set_fact:
+        timestamp: "{{ ansible_date_time.iso8601_basic_short }}"
+
+    - name: Create tar.gz archive
+      archive:
+        path: "{{ backup_temp_dir.path }}/"
+        dest: "{{ backup_dir }}/k8s_backup_{{ timestamp }}.tar.gz"
+        format: gz
+
+    - name: Create checksum file
+      block:
+        - name: Calculate checksum
+          stat:
+            path: "{{ backup_dir }}/k8s_backup_{{ timestamp }}.tar.gz"
+            get_attributes: false
+            get_checksum: true
+            get_mime: false
+            checksum_algorithm: sha1
+          register: stat_k8s_backup_tar_gz
+
+        - name: Save checksum to a file
+          copy:
+            dest: "{{ stat_k8s_backup_tar_gz.stat.path }}.sha1"
+            content: |
+              {{ stat_k8s_backup_tar_gz.stat.checksum }}  {{ stat_k8s_backup_tar_gz.stat.path | basename }}