From 653abed35bd586d60124a81040eed167742e697d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Opala?= Date: Mon, 25 May 2020 13:02:20 +0200 Subject: [PATCH] backup/recovery: adding rabbitmq (without messages) (#1291) * backup/recovery: adding rabbitmq (without messages) * backup/recovery: adding rabbitmq (archive fix) --- core/src/epicli/cli/epicli.py | 4 +- .../ansible/playbooks/backup_rabbitmq.yml | 17 ++++ .../ansible/playbooks/recovery_rabbitmq.yml | 24 ++++++ .../tasks/rabbitmq_rabbitmq_definitions.yml | 78 ++++++++++++++++++ .../backup/tasks/rabbitmq_rabbitmq_etc.yml | 52 ++++++++++++ .../tasks/rabbitmq_rabbitmq_definitions.yml | 80 +++++++++++++++++++ .../recovery/tasks/rabbitmq_rabbitmq_etc.yml | 76 ++++++++++++++++++ 7 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 core/src/epicli/data/common/ansible/playbooks/backup_rabbitmq.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/recovery_rabbitmq.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_definitions.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_etc.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_definitions.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_etc.yml diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py index 685bb27e66..7505ec9716 100644 --- a/core/src/epicli/cli/epicli.py +++ b/core/src/epicli/cli/epicli.py @@ -285,7 +285,7 @@ def backup_parser(subparsers): sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True, help='Absolute path to directory with build artifacts.') - available_components = {'kubernetes', 'loadbalancer', 'logging', 'monitoring', 'postgresql'} + available_components = {'kubernetes', 'loadbalancer', 'logging', 'monitoring', 'postgresql', 'rabbitmq'} enabled_components = set(available_components) # enable everything by default enabled_components_joined = ','.join(sorted(enabled_components)) @@ -309,7 +309,7 @@ def recovery_parser(subparsers): sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True, help='Absolute path to directory with build artifacts.') - available_components = {'kubernetes', 'loadbalancer', 'logging', 'monitoring', 'postgresql'} + available_components = {'kubernetes', 'loadbalancer', 'logging', 'monitoring', 'postgresql', 'rabbitmq'} enabled_components = set() # disable everything by default enabled_components_joined = ','.join(sorted(enabled_components)) diff --git a/core/src/epicli/data/common/ansible/playbooks/backup_rabbitmq.yml b/core/src/epicli/data/common/ansible/playbooks/backup_rabbitmq.yml new file mode 100644 index 0000000000..aae5320032 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/backup_rabbitmq.yml @@ -0,0 +1,17 @@ +--- +# Ansible playbook for backing up rabbitmq config + +- hosts: rabbitmq[0] + gather_facts: true + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: backup + tasks_from: rabbitmq_rabbitmq_definitions + - import_role: + name: backup + tasks_from: rabbitmq_rabbitmq_etc + vars_files: + - roles/rabbitmq/vars/main.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/recovery_rabbitmq.yml b/core/src/epicli/data/common/ansible/playbooks/recovery_rabbitmq.yml new file mode 100644 index 0000000000..c5e0c903e1 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/recovery_rabbitmq.yml @@ -0,0 +1,24 @@ +--- +# Ansible playbook for recovering rabbitmq config + +- hosts: rabbitmq + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: rabbitmq_rabbitmq_etc + vars_files: + - roles/rabbitmq/vars/main.yml + +- hosts: rabbitmq[0] + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: rabbitmq_rabbitmq_definitions + vars_files: + - roles/rabbitmq/vars/main.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_definitions.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_definitions.yml new file mode 100644 index 0000000000..597540b1b8 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_definitions.yml @@ -0,0 +1,78 @@ +--- +- name: Set helper facts + set_fact: + snapshot_name: >- + {{ ansible_date_time.iso8601_basic_short | replace('T','-') }} + +- debug: var=snapshot_name + +- name: Ensure management api is enabled + shell: | + rabbitmq-plugins enable rabbitmq_management + args: + executable: /bin/bash + +- name: Ensure the rabbitmqadmin binary is installed + shell: | + curl -fsSL http://localhost:15672/cli/rabbitmqadmin \ + -o /usr/local/bin/rabbitmqadmin \ + && chmod +x /usr/local/bin/rabbitmqadmin + args: + creates: /usr/local/bin/rabbitmqadmin + executable: /bin/bash + +- name: Ensure a folder to hold definitions in exists + file: + path: /var/lib/rabbitmq/definitions/ + state: directory + +- name: Store definitions json file + shell: | + rabbitmqadmin export /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json + args: + executable: /bin/bash + +- name: Create and copy definitions archive to backup destination + always: + - name: Delete definitions archive (cleanup) + file: + path: "{{ item }}" + state: absent + loop: + - "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz.sha1" + + block: + - name: Ensure backup dir exists + file: + path: "{{ backup_dir }}/" + state: directory + + - name: Create definitions archive + archive: + dest: "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz" + path: /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json + format: gz + force_archive: true + + - name: Calculate checksum from definitions archive + stat: + path: "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_rabbitmq_definitions_archive + + - name: Store definitions archive checksum in a file + copy: + dest: "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz.sha1" + content: | + {{ stat_rabbitmq_definitions_archive.stat.checksum }} rabbitmq_definitions_{{ snapshot_name }}.tar.gz + + - name: Transfer definitions archive via rsync + import_tasks: download_via_rsync.yml + vars: + artifacts: + - "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/rabbitmq_definitions_{{ snapshot_name }}.tar.gz.sha1" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_etc.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_etc.yml new file mode 100644 index 0000000000..ebee94e0dc --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/rabbitmq_rabbitmq_etc.yml @@ -0,0 +1,52 @@ +--- +- name: Assert that the "snapshot_name" fact is defined and valid + assert: + that: + - snapshot_name is defined + - snapshot_name is string + - snapshot_name | length > 0 + fail_msg: The "snapshot_name" fact must be defined and must be a non-empty string. + +- name: Create and copy etc archive to backup destination + always: + - name: Delete etc archive (cleanup) + file: + path: "{{ item }}" + state: absent + loop: + - "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz.sha1" + + block: + - name: Ensure backup dir exists + file: + path: "{{ backup_dir }}/" + state: directory + + - name: Create etc archive + archive: + dest: "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz" + path: /etc/rabbitmq/ # keep the / here! + format: gz + + - name: Calculate checksum from etc archive + stat: + path: "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_rabbitmq_etc_archive + + - name: Store etc archive checksum in a file + copy: + dest: "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz.sha1" + content: | + {{ stat_rabbitmq_etc_archive.stat.checksum }} rabbitmq_etc_{{ snapshot_name }}.tar.gz + + - name: Transfer etc archive via rsync + import_tasks: download_via_rsync.yml + vars: + artifacts: + - "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/rabbitmq_etc_{{ snapshot_name }}.tar.gz.sha1" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_definitions.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_definitions.yml new file mode 100644 index 0000000000..6bc5dfb6e3 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_definitions.yml @@ -0,0 +1,80 @@ +--- +- name: Find all rabbitmq etc archives + delegate_to: "{{ recovery_source_host }}" + find: + paths: "{{ recovery_source_dir }}/" + patterns: "rabbitmq_definitions_*-*.tar.gz" + file_type: file + recurse: false + register: find_rabbitmq_definitions_archives + +- name: Do sanity check if there are any etc archives available + assert: + that: find_rabbitmq_definitions_archives.matched > 0 + fail_msg: No etc archives found. + +- name: Pick the newest etc archive + set_fact: + newest_definitions_archive_path: >- + {{ find_rabbitmq_definitions_archives.files | map(attribute='path') | max }} + +- name: Transfer etc archive via rsync + import_tasks: upload_via_rsync.yml + vars: + artifacts: + - "{{ newest_definitions_archive_path }}" + - "{{ newest_definitions_archive_path }}.sha1" + +- name: Slurp etc archive checksum from file + slurp: + path: "{{ recovery_dir }}/{{ newest_definitions_archive_path | basename }}.sha1" + register: slurp_definitions_archive_checksum + +- name: Calculate checksum from etc archive + stat: + path: "{{ recovery_dir }}/{{ newest_definitions_archive_path | basename }}" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_definitions_archive + +- name: Compare etc archive checksums + assert: + that: (slurp_definitions_archive_checksum.content | b64decode | trim).startswith(stat_definitions_archive.stat.checksum) + fail_msg: Checksums do not match. + +- name: Ensure a folder to hold definitions in exists + file: + path: /var/lib/rabbitmq/definitions/ + state: directory + +- name: Extract etc archive to etc directory + unarchive: + dest: /var/lib/rabbitmq/definitions/ + src: "{{ recovery_dir }}/{{ newest_definitions_archive_path | basename }}" + remote_src: true + +- name: Ensure management api is enabled + shell: | + rabbitmq-plugins enable rabbitmq_management + args: + executable: /bin/bash + +- name: Ensure the rabbitmqadmin binary is installed + shell: | + curl -fsSL http://localhost:15672/cli/rabbitmqadmin \ + -o /usr/local/bin/rabbitmqadmin \ + && chmod +x /usr/local/bin/rabbitmqadmin + args: + creates: /usr/local/bin/rabbitmqadmin + executable: /bin/bash + +- name: Import definitions json file + shell: | + rabbitmqadmin import /var/lib/rabbitmq/definitions/definitions-{{ snapshot_name }}.json + args: + executable: /bin/bash + vars: + snapshot_name: >- + {{ newest_definitions_archive_path | basename | regex_replace('^rabbitmq_definitions_(.*).tar.gz$', '\1') }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_etc.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_etc.yml new file mode 100644 index 0000000000..bd72cdee5d --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/rabbitmq_rabbitmq_etc.yml @@ -0,0 +1,76 @@ +--- +- name: Find all rabbitmq etc archives + delegate_to: "{{ recovery_source_host }}" + find: + paths: "{{ recovery_source_dir }}/" + patterns: "rabbitmq_etc_*-*.tar.gz" + file_type: file + recurse: false + register: find_rabbitmq_etc_archives + +- name: Do sanity check if there are any etc archives available + assert: + that: find_rabbitmq_etc_archives.matched > 0 + fail_msg: No etc archives found. + +- name: Pick the newest etc archive + set_fact: + newest_etc_archive_path: >- + {{ find_rabbitmq_etc_archives.files | map(attribute='path') | max }} + +- name: Transfer etc archive via rsync + import_tasks: upload_via_rsync.yml + vars: + artifacts: + - "{{ newest_etc_archive_path }}" + - "{{ newest_etc_archive_path }}.sha1" + +- name: Slurp etc archive checksum from file + slurp: + path: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}.sha1" + register: slurp_etc_archive_checksum + +- name: Calculate checksum from etc archive + stat: + path: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_etc_archive + +- name: Compare etc archive checksums + assert: + that: (slurp_etc_archive_checksum.content | b64decode | trim).startswith(stat_etc_archive.stat.checksum) + fail_msg: Checksums do not match. + +- name: Stop rabbitmq service + systemd: + name: rabbitmq-server + state: stopped + +- name: Find everything in the etc directory + find: + paths: /etc/rabbitmq/ + patterns: "*" + file_type: any + recurse: false + register: find_everything_in_etc_directory + +- name: Remove all rabbitmq etc config files + file: + path: "{{ item }}" + state: absent + loop: >- + {{ find_everything_in_etc_directory.files | map(attribute='path') | list }} + +- name: Extract etc archive to etc directory + unarchive: + dest: /etc/rabbitmq/ + src: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}" + remote_src: true + +- name: Start rabbitmq service + systemd: + name: rabbitmq-server + state: started