From 8b385331bc9cc13073ce2b983707b444f674c266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Opala?= Date: Fri, 8 May 2020 12:49:00 +0200 Subject: [PATCH] backup/recovery: uncommenting and refactoring legacy code (#1241) --- core/src/epicli/cli/engine/PatchEngine.py | 21 ++-- core/src/epicli/cli/epicli.py | 43 +++++++- .../data/common/ansible/playbooks/backup.yml | 9 -- .../ansible/playbooks/backup_kubernetes.yml | 11 ++ .../ansible/playbooks/backup_monitoring.yml | 11 ++ .../ansible/playbooks/backup_postgresql.yml | 11 ++ .../common/ansible/playbooks/recovery.yml | 9 -- .../ansible/playbooks/recovery_kubernetes.yml | 11 ++ .../ansible/playbooks/recovery_monitoring.yml | 11 ++ .../ansible/playbooks/recovery_postgresql.yml | 11 ++ .../playbooks/roles/backup/defaults/main.yml | 2 +- .../playbooks/roles/backup/meta/main.yml | 3 - .../roles/backup/tasks/kubernetes.yml | 104 ++++++++++++++++++ .../playbooks/roles/backup/tasks/main.yml | 69 ------------ .../roles/backup/tasks/monitoring.yml | 1 + .../roles/backup/tasks/postgresql.yml | 1 + .../roles/recovery/defaults/main.yml | 2 +- .../tasks/{main.yml => kubernetes.yml} | 14 ++- .../roles/recovery/tasks/monitoring.yml | 1 + .../roles/recovery/tasks/postgresql.yml | 1 + 20 files changed, 237 insertions(+), 109 deletions(-) delete mode 100644 core/src/epicli/data/common/ansible/playbooks/backup.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/backup_kubernetes.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/backup_monitoring.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/backup_postgresql.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/recovery.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/recovery_kubernetes.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/recovery_monitoring.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/recovery_postgresql.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/meta/main.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/kubernetes.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/main.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/monitoring.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml rename core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/{main.yml => kubernetes.yml} (93%) create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/monitoring.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml diff --git a/core/src/epicli/cli/engine/PatchEngine.py b/core/src/epicli/cli/engine/PatchEngine.py index 0d08e77b5f..28aece5d41 100644 --- a/core/src/epicli/cli/engine/PatchEngine.py +++ b/core/src/epicli/cli/engine/PatchEngine.py @@ -11,6 +11,7 @@ class PatchEngine(Step): def __init__(self, input_data): super().__init__(__name__) self.build_directory = input_data.build_directory + self.components = input_data.components self.ansible_command = AnsibleCommand() def __enter__(self): @@ -21,15 +22,17 @@ def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) def backup(self): - self.upgrade_patch_files_and_run('backup') + for component in sorted(self.components): + self.upgrade_patch_files_and_run('backup', component) return 0 def recovery(self): - self.upgrade_patch_files_and_run('recovery') + for component in sorted(self.components): + self.upgrade_patch_files_and_run('recovery', component) return 0 - def upgrade_patch_files_and_run(self, action): - self.logger.info(f'Running {action}...') + def upgrade_patch_files_and_run(self, action, component): + self.logger.info(f'Running {action} on {component}...') #copy role files roles_build_path = os.path.join(self.build_directory, 'ansible/roles', action) @@ -37,10 +40,10 @@ def upgrade_patch_files_and_run(self, action): copy_files_recursively(roles_source_path, roles_build_path) #copy playbook file - playbook_build_path = os.path.join(self.build_directory, 'ansible/') + action + '.yml' - playbook_source_path = os.path.join(AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH) + action + '.yml' - copy_file(playbook_source_path, playbook_build_path) - + playbook_build_path = os.path.join(self.build_directory, 'ansible/') + action + '_' + component + '.yml' + playbook_source_path = os.path.join(AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH) + action + '_' + component + '.yml' + copy_file(playbook_source_path, playbook_build_path) + #run the playbook inventory_path = get_inventory_path_for_build(self.build_directory) - self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=playbook_build_path) \ No newline at end of file + self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=playbook_build_path) diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py index e4d5750250..799e23e945 100644 --- a/core/src/epicli/cli/epicli.py +++ b/core/src/epicli/cli/epicli.py @@ -92,12 +92,11 @@ def debug_level(x): upgrade_parser(subparsers) delete_parser(subparsers) test_parser(subparsers) - ''' validate_parser(subparsers) + ''' backup_parser(subparsers) recovery_parser(subparsers) - ''' # check if there were any variables and display full help if len(sys.argv) < 2: @@ -260,6 +259,24 @@ def run_validate(args): return engine.validate() sub_parser.set_defaults(func=run_validate) +''' + + +def _component_parser_for(available_components={}): + def parse_components(value): + parsed_items = set( + item_stripped + for item in value.split(',') + for item_stripped in [item.strip()] + if item_stripped + ) + if len(parsed_items) == 1 and 'all' in parsed_items: + return set(available_components) + difference = parsed_items - set(available_components) + if difference: + raise Exception('Error parsing components: invalid values present') + return parsed_items + return parse_components def backup_parser(subparsers): @@ -268,6 +285,15 @@ def backup_parser(subparsers): sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True, help='Absolute path to directory with build artifacts.') + available_components = {'kubernetes', 'postgresql', 'monitoring'} + + enabled_components = set(available_components) # enable everything by default + enabled_components_joined = ','.join(sorted(enabled_components)) + + sub_parser.add_argument('-c', '--components', dest='components', type=_component_parser_for(available_components), required=False, + help=f'Specify comma-separated list of components to backup (defaults to "{enabled_components_joined}").', + default=enabled_components_joined) + def run_backup(args): experimental_query() adjust_paths_from_build(args) @@ -278,10 +304,20 @@ def run_backup(args): def recovery_parser(subparsers): - sub_parser = subparsers.add_parser('recovery', description='[Experimental]: Recover from existing backup.') + sub_parser = subparsers.add_parser('recovery', + description='[Experimental]: Recover from existing backup.') sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True, help='Absolute path to directory with build artifacts.') + available_components = {'kubernetes', 'postgresql', 'monitoring'} + + enabled_components = set() # disable everything by default + enabled_components_joined = ','.join(sorted(enabled_components)) + + sub_parser.add_argument('-c', '--components', dest='components', type=_component_parser_for(available_components), required=False, + help=f'Specify comma-separated list of components to restore (defaults to "{enabled_components_joined}").', + default=enabled_components_joined) + def run_recovery(args): experimental_query() adjust_paths_from_build(args) @@ -289,7 +325,6 @@ def run_recovery(args): return engine.recovery() sub_parser.set_defaults(func=run_recovery) -''' def experimental_query(): diff --git a/core/src/epicli/data/common/ansible/playbooks/backup.yml b/core/src/epicli/data/common/ansible/playbooks/backup.yml deleted file mode 100644 index 35575988d4..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/backup.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# Ansible playbook for backing up Kubernetes cluster - -- hosts: kubernetes_master - serial: 1 - become: true - become_method: sudo - roles: - - backup diff --git a/core/src/epicli/data/common/ansible/playbooks/backup_kubernetes.yml b/core/src/epicli/data/common/ansible/playbooks/backup_kubernetes.yml new file mode 100644 index 0000000000..5e2c8e0230 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/backup_kubernetes.yml @@ -0,0 +1,11 @@ +--- +# Ansible playbook for backing up Kubernetes cluster + +- hosts: kubernetes_master[0] + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: backup + tasks_from: kubernetes diff --git a/core/src/epicli/data/common/ansible/playbooks/backup_monitoring.yml b/core/src/epicli/data/common/ansible/playbooks/backup_monitoring.yml new file mode 100644 index 0000000000..6e3b18c0da --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/backup_monitoring.yml @@ -0,0 +1,11 @@ +--- +# Ansible playbook for backing up monitoring data + +- hosts: prometheus + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: backup + tasks_from: monitoring diff --git a/core/src/epicli/data/common/ansible/playbooks/backup_postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/backup_postgresql.yml new file mode 100644 index 0000000000..25e4eb4c87 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/backup_postgresql.yml @@ -0,0 +1,11 @@ +--- +# Ansible playbook for backing up Postgresql database + +- hosts: postgresql[0] + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: backup + tasks_from: postgresql diff --git a/core/src/epicli/data/common/ansible/playbooks/recovery.yml b/core/src/epicli/data/common/ansible/playbooks/recovery.yml deleted file mode 100644 index 25281a0ae3..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/recovery.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# Ansible playbook for recovering Kubernetes cluster - -- hosts: kubernetes_master - serial: 1 - become: true - become_method: sudo - roles: - - recovery diff --git a/core/src/epicli/data/common/ansible/playbooks/recovery_kubernetes.yml b/core/src/epicli/data/common/ansible/playbooks/recovery_kubernetes.yml new file mode 100644 index 0000000000..182512690f --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/recovery_kubernetes.yml @@ -0,0 +1,11 @@ +--- +# Ansible playbook for recovering Kubernetes cluster + +- hosts: kubernetes_master[0] + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: kubernetes diff --git a/core/src/epicli/data/common/ansible/playbooks/recovery_monitoring.yml b/core/src/epicli/data/common/ansible/playbooks/recovery_monitoring.yml new file mode 100644 index 0000000000..fdb65466fd --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/recovery_monitoring.yml @@ -0,0 +1,11 @@ +--- +# Ansible playbook for recovering monitoring data + +- hosts: prometheus + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: monitoring diff --git a/core/src/epicli/data/common/ansible/playbooks/recovery_postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/recovery_postgresql.yml new file mode 100644 index 0000000000..052e9ea1a5 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/recovery_postgresql.yml @@ -0,0 +1,11 @@ +--- +# Ansible playbook for recovering Postgresql database + +- hosts: postgresql[0] + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: postgresql diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml index 7ae2f6b6dd..f86c7fac79 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml @@ -1,2 +1,2 @@ --- -backup_dir: /home/{{ admin_user.name }}/backupdir +backup_dir: /epibackup diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/meta/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/meta/main.yml deleted file mode 100644 index 745ba4d956..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/backup/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: preflight_facts diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/kubernetes.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/kubernetes.yml new file mode 100644 index 0000000000..8cb45da6d5 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/kubernetes.yml @@ -0,0 +1,104 @@ +--- +- name: Ensure backup directory exists + file: + path: "{{ backup_dir }}/" + state: directory + mode: u=rwx,go= + +- name: Create temporary directory + tempfile: + path: "{{ backup_dir }}/" + suffix: .tmp + state: directory + register: backup_temp_dir + +- name: Save backup and cleanup afterwards + always: + - name: Delete temporary directory + file: + path: "{{ backup_temp_dir.path }}/" + state: absent + block: + - name: Get etcd image name + shell: | + kubectl get pods \ + --all-namespaces \ + --output jsonpath={{ jsonpath }} + vars: + jsonpath: >- + "{.items[*].spec.containers[?(@.name=='etcd')].image}" + environment: + KUBECONFIG: /etc/kubernetes/admin.conf + register: etcd_image_name + + - name: Save etcd image name to a file + copy: + dest: "{{ backup_temp_dir.path }}/etcd_ver.txt" + content: |- + {{ etcd_image_name.stdout | trim }} + + - name: Save kubernetes PKI + copy: + src: /etc/kubernetes/pki # do not put / at the end here! + dest: "{{ backup_temp_dir.path }}/" + remote_src: true + + - name: Save etcd snapshot + shell: | + docker run \ + -v "{{ backup_temp_dir.path }}/:/backup/" \ + --network host \ + --env ETCDCTL_API=3 \ + --rm "{{ etcd_image_name.stdout | trim }}" \ + etcdctl \ + --endpoints https://127.0.0.1:2379 \ + --cacert /backup/pki/etcd/ca.crt \ + --cert /backup/pki/etcd/healthcheck-client.crt \ + --key /backup/pki/etcd/healthcheck-client.key \ + snapshot save /backup/etcd-snapshot.db + args: + executable: /bin/bash + + - name: Check if kubeadm configuration file exists + stat: + path: /etc/kubeadm/kubeadm-config.yml + get_attributes: false + get_checksum: false + get_mime: false + register: stat_kubeadm_config_yml + + - when: stat_kubeadm_config_yml.stat.exists + block: + - name: Save kubeadm configuration file + copy: + src: "{{ stat_kubeadm_config_yml.stat.path }}" + dest: "{{ backup_temp_dir.path }}/" + remote_src: true + + - name: Get current timestamp + run_once: true + set_fact: + timestamp: "{{ ansible_date_time.iso8601_basic_short }}" + + - name: Create tar.gz archive + archive: + path: "{{ backup_temp_dir.path }}/" + dest: "{{ backup_dir }}/k8s_backup_{{ timestamp }}.tar.gz" + format: gz + + - name: Create checksum file + block: + - name: Calculate checksum + stat: + path: "{{ backup_dir }}/k8s_backup_{{ timestamp }}.tar.gz" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_k8s_backup_tar_gz + + - name: Save checksum to a file + copy: + dest: "{{ stat_k8s_backup_tar_gz.stat.path }}.sha1" + content: | + {{ stat_k8s_backup_tar_gz.stat.checksum }} {{ stat_k8s_backup_tar_gz.stat.path | basename }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/main.yml deleted file mode 100644 index 582290f177..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/main.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- -- name: Create a backup directory - file: - path: "{{ backup_dir }}" - state: directory - -# Ansible 2.8 -# - name: Backup certificates -# copy: -# src: /etc/kubernetes/pki -# dest: "{{ backup_dir }}/tmp" -# remote_src: yes - -# Ansible 2.7 -- name: Backup certificates - synchronize: - src: /etc/kubernetes/pki - dest: "{{ backup_dir }}/tmp" - recursive: yes - delegate_to: "{{ inventory_hostname }}" - -- name: Get etcd image name - environment: - KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config" - shell: kubectl get pods --all-namespaces -o=jsonpath="{.items[*].spec.containers[?(@.name=='etcd')].image}" - register: etcd_image_name - -- name: Save etcd image name to file - copy: - content: "{{ etcd_image_name.stdout }}" - dest: "{{ backup_dir }}/tmp/etcd_ver.txt" - -- name: Create etcd snapshot - shell: > - docker run -v "{{ backup_dir }}/tmp":/backup \ - --network host \ - --env ETCDCTL_API=3 \ - --rm {{ etcd_image_name.stdout }} \ - etcdctl --endpoints=https://127.0.0.1:2379 \ - --cacert=/backup/pki/etcd/ca.crt \ - --cert=/backup/pki/etcd/healthcheck-client.crt \ - --key=/backup/pki/etcd/healthcheck-client.key \ - snapshot save /backup/etcd-snapshot.db - -- name: Check if kubeadm configuration file exists - stat: - path: /etc/kubeadm/kubeadm-config.yml - register: stat_result - -- name: Backup kubeadm configuration file - copy: - src: /etc/kubeadm/kubeadm-config.yml - dest: "{{ backup_dir }}/tmp" - remote_src: yes - when: stat_result.stat.exists - -- name: Set variable with current timestamp - set_fact: timestamp="{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" - -- name: Create a tar gz archive - archive: - path: "{{ backup_dir }}/tmp/" - dest: "{{ backup_dir }}/k8s_backup_{{ timestamp }}.tar.gz" - format: gz - -- name: Clean temporary directory - file: - state: absent - path: "{{ backup_dir }}/tmp/" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/monitoring.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/monitoring.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/monitoring.yml @@ -0,0 +1 @@ +--- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml @@ -0,0 +1 @@ +--- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml index 7ae2f6b6dd..f86c7fac79 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml @@ -1,2 +1,2 @@ --- -backup_dir: /home/{{ admin_user.name }}/backupdir +backup_dir: /epibackup diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/kubernetes.yml similarity index 93% rename from core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/main.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/kubernetes.yml index cea8d59c93..80b7dc1619 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/kubernetes.yml @@ -1,4 +1,10 @@ --- + +- fail: + msg: This is a dangerous procedure that is currently disabled because it needs refactoring! + +# The procedure of restoring kubernetes cluster + - name: Reset kubeadm shell: kubeadm reset -f @@ -49,7 +55,7 @@ state: directory - name: Get etcd image name - shell: cat "{{ backup_dir }}/tmp/etcd_ver.txt" + shell: cat "{{ backup_dir }}/tmp/etcd_ver.txt" register: etcd_image_name - name: Restore etcd backup @@ -95,7 +101,7 @@ retries: 120 delay: 10 -- name: Check cluster version +- name: Check cluster version environment: KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config" shell: kubectl version --short | grep -i server @@ -109,7 +115,7 @@ - name: Show upgrade plan shell: kubeadm upgrade plan when: '"1.13" in cluster_version.stdout' - + rescue: - name: Find the existing etcd server certificates find: @@ -125,7 +131,7 @@ - name: Regenerate the etcd server certificates shell: kubeadm init phase certs etcd-server - + - name: Clean temporary directory file: state: absent diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/monitoring.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/monitoring.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/monitoring.yml @@ -0,0 +1 @@ +--- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml @@ -0,0 +1 @@ +---