From a83eb75d4615b05d67b9a1edb8ee1120911bac6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Opala?= Date: Wed, 20 May 2020 11:06:38 +0200 Subject: [PATCH] backup/recovery: logging (WIP) (#1277) - work in progress, does not work with elasticsearch clusters yet --- core/src/epicli/cli/epicli.py | 4 +- .../ansible/playbooks/backup_logging.yml | 29 +++++ .../ansible/playbooks/recovery_logging.yml | 28 +++++ .../playbooks/roles/backup/defaults/main.yml | 2 + .../tasks/logging_elasticsearch_etc.yml | 52 ++++++++ .../tasks/logging_elasticsearch_snapshot.yml | 111 +++++++++++++++++ .../roles/backup/tasks/logging_kibana_etc.yml | 52 ++++++++ .../tasks/configure-es.yml | 9 +- .../templates/elasticsearch.yml.j2 | 8 +- .../roles/recovery/defaults/main.yml | 2 + .../tasks/logging_elasticsearch_etc.yml | 76 ++++++++++++ .../tasks/logging_elasticsearch_snapshot.yml | 113 ++++++++++++++++++ .../recovery/tasks/logging_kibana_etc.yml | 76 ++++++++++++ .../common/defaults/configuration/logging.yml | 1 + 14 files changed, 558 insertions(+), 5 deletions(-) create mode 100644 core/src/epicli/data/common/ansible/playbooks/backup_logging.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/recovery_logging.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py index 5c361f42ad..685bb27e66 100644 --- a/core/src/epicli/cli/epicli.py +++ b/core/src/epicli/cli/epicli.py @@ -285,7 +285,7 @@ def backup_parser(subparsers): sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True, help='Absolute path to directory with build artifacts.') - available_components = {'kubernetes', 'loadbalancer', 'postgresql', 'monitoring'} + available_components = {'kubernetes', 'loadbalancer', 'logging', 'monitoring', 'postgresql'} enabled_components = set(available_components) # enable everything by default enabled_components_joined = ','.join(sorted(enabled_components)) @@ -309,7 +309,7 @@ def recovery_parser(subparsers): sub_parser.add_argument('-b', '--build', dest='build_directory', type=str, required=True, help='Absolute path to directory with build artifacts.') - available_components = {'kubernetes', 'loadbalancer', 'postgresql', 'monitoring'} + available_components = {'kubernetes', 'loadbalancer', 'logging', 'monitoring', 'postgresql'} enabled_components = set() # disable everything by default enabled_components_joined = ','.join(sorted(enabled_components)) diff --git a/core/src/epicli/data/common/ansible/playbooks/backup_logging.yml b/core/src/epicli/data/common/ansible/playbooks/backup_logging.yml new file mode 100644 index 0000000000..2fe1f13e12 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/backup_logging.yml @@ -0,0 +1,29 @@ +--- +# Ansible playbook for backing up logging data + +- hosts: logging[0] + gather_facts: true + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: backup + tasks_from: logging_elasticsearch_snapshot + - import_role: + name: backup + tasks_from: logging_elasticsearch_etc + vars_files: + - roles/logging/vars/main.yml + +- hosts: kibana[0] + gather_facts: true + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: backup + tasks_from: logging_kibana_etc + vars_files: + - roles/kibana/vars/main.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/recovery_logging.yml b/core/src/epicli/data/common/ansible/playbooks/recovery_logging.yml new file mode 100644 index 0000000000..73878915b1 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/recovery_logging.yml @@ -0,0 +1,28 @@ +--- +# Ansible playbook for recovering logging data + +- hosts: logging[0] + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: logging_elasticsearch_etc + - import_role: + name: recovery + tasks_from: logging_elasticsearch_snapshot + vars_files: + - roles/logging/vars/main.yml + +- hosts: kibana[0] + gather_facts: true + become: true + become_method: sudo + serial: 1 + tasks: + - import_role: + name: recovery + tasks_from: logging_kibana_etc + vars_files: + - roles/kibana/vars/main.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml index 9cc69f8704..c21c1bcd99 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/defaults/main.yml @@ -2,3 +2,5 @@ backup_dir: /epibackup backup_destination_dir: "{{ backup_dir }}/mounted" backup_destination_host: "{{ groups.kubernetes_master.0 }}" +elasticsearch_snapshot_repository_name: epiphany +elasticsearch_snapshot_repository_location: /var/lib/elasticsearch-snapshots diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml new file mode 100644 index 0000000000..e5fe2fbae6 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_etc.yml @@ -0,0 +1,52 @@ +--- +- name: Assert that the "snapshot_name" fact is defined and valid + assert: + that: + - snapshot_name is defined + - snapshot_name is string + - snapshot_name | length > 0 + fail_msg: The "snapshot_name" fact must be defined and must be a non-empty string. + +- name: Create and copy etc archive to backup destination + always: + - name: Delete etc archive (cleanup) + file: + path: "{{ item }}" + state: absent + loop: + - "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz.sha1" + + block: + - name: Ensure backup dir exists + file: + path: "{{ backup_dir }}/" + state: directory + + - name: Create etc archive + archive: + dest: "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz" + path: /etc/elasticsearch/ # keep the / here! + format: gz + + - name: Calculate checksum from etc archive + stat: + path: "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_elasticsearch_etc_archive + + - name: Store etc archive checksum in a file + copy: + dest: "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz.sha1" + content: | + {{ stat_elasticsearch_etc_archive.stat.checksum }} elasticsearch_etc_{{ snapshot_name }}.tar.gz + + - name: Transfer etc archive via rsync + import_tasks: download_via_rsync.yml + vars: + artifacts: + - "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/elasticsearch_etc_{{ snapshot_name }}.tar.gz.sha1" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml new file mode 100644 index 0000000000..50719b6858 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_elasticsearch_snapshot.yml @@ -0,0 +1,111 @@ +--- +- name: Set helper facts + set_fact: + elasticsearch_endpoint: >- + https://{{ ansible_default_ipv4.address }}:9200 + snapshot_name: >- + {{ ansible_date_time.iso8601_basic_short | replace('T','-') }} + vars: + uri_template: &uri + client_cert: /etc/elasticsearch/kirk.pem + client_key: /etc/elasticsearch/kirk-key.pem + validate_certs: false + body_format: json + +- debug: var=snapshot_name + +- name: Check cluster health + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_cluster/health" + method: GET + register: uri_response + until: uri_response is success + retries: 12 + delay: 5 + +- name: Ensure snapshot repository is defined + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}" + method: PUT + body: + type: fs + settings: + location: "{{ elasticsearch_snapshot_repository_location }}" + compress: true + +- name: Trigger snapshot creation + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ snapshot_name }}" + method: PUT + +- name: Wait (up to 12h) for snapshot completion + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ snapshot_name }}" + method: GET + register: uri_response + until: (uri_response.json.snapshots | selectattr('snapshot', 'equalto', snapshot_name) | first).state == "SUCCESS" + retries: "{{ (12 * 3600 // 10) | int }}" # 12h + delay: 10 + +- name: Find all snapshots + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/_all" + method: GET + register: uri_response + +- name: Delete old snapshots + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ item }}" + method: DELETE + loop: >- + {{ uri_response.json.snapshots | map(attribute='snapshot') | reject('equalto', snapshot_name) | list }} + +- name: Create and copy snapshot archive to backup destination + always: + - name: Delete snapshot archive (cleanup) + file: + path: "{{ item }}" + state: absent + loop: + - "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz.sha1" + + block: + - name: Ensure backup dir exists + file: + path: "{{ backup_dir }}/" + state: directory + + - name: Create snapshot archive + archive: + dest: "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz" + path: "{{ elasticsearch_snapshot_repository_location }}/*" + format: gz + + - name: Calculate checksum from snapshot archive + stat: + path: "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_snapshot_archive + + - name: Store snapshot archive checksum in a file + copy: + dest: "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz.sha1" + content: | + {{ stat_snapshot_archive.stat.checksum }} elasticsearch_snapshot_{{ snapshot_name }}.tar.gz + + - name: Transfer snapshot archive via rsync + import_tasks: download_via_rsync.yml + vars: + artifacts: + - "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/elasticsearch_snapshot_{{ snapshot_name }}.tar.gz.sha1" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml new file mode 100644 index 0000000000..911d544fcd --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/logging_kibana_etc.yml @@ -0,0 +1,52 @@ +--- +- name: Assert that the "snapshot_name" fact is defined and valid + assert: + that: + - snapshot_name is defined + - snapshot_name is string + - snapshot_name | length > 0 + fail_msg: The "snapshot_name" fact must be defined and must be a non-empty string. + +- name: Create and copy etc archive to backup destination + always: + - name: Delete etc archive (cleanup) + file: + path: "{{ item }}" + state: absent + loop: + - "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz.sha1" + + block: + - name: Ensure backup dir exists + file: + path: "{{ backup_dir }}/" + state: directory + + - name: Create etc archive + archive: + dest: "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz" + path: /etc/kibana/ # keep the / here! + format: gz + + - name: Calculate checksum from etc archive + stat: + path: "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_etc_archive + + - name: Store etc archive checksum in a file + copy: + dest: "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz.sha1" + content: | + {{ stat_etc_archive.stat.checksum }} kibana_etc_{{ snapshot_name }}.tar.gz + + - name: Transfer etc archive via rsync + import_tasks: download_via_rsync.yml + vars: + artifacts: + - "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz" + - "{{ backup_dir }}/kibana_etc_{{ snapshot_name }}.tar.gz.sha1" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml index c0c6d95729..6f73e83724 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml @@ -1,4 +1,11 @@ --- +- name: Ensure snapshot folder exists + file: + path: "{{ specification.paths.repo }}/" + state: directory + owner: elasticsearch + group: elasticsearch + mode: u=rwx,go= - name: Create Elasticsearch configuration file template: @@ -19,4 +26,4 @@ systemd: name: elasticsearch state: started - enabled: yes \ No newline at end of file + enabled: yes diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 index 49eb652f2d..b18e3a8cb7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/elasticsearch.yml.j2 @@ -32,6 +32,10 @@ node.name: {{ansible_hostname}} # path.data: {{specification.paths.data}} # +# Path to directory where to store the data: +# +path.repo: {{specification.paths.repo}} +# # Path to log files: # path.logs: {{specification.paths.logs}} @@ -52,7 +56,7 @@ path.logs: {{specification.paths.logs}} # # Set the bind address to a specific IP (IPv4 or IPv6): # -network.host: {{ansible_hostname}} +network.host: {{ansible_hostname}} # # Set a custom port for HTTP: # @@ -113,4 +117,4 @@ opendistro_security.check_snapshot_restore_write_privileges: true opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] cluster.routing.allocation.disk.threshold_enabled: false node.max_local_storage_nodes: 3 -######## End OpenDistro for Elasticsearch Security Demo Configuration ######## \ No newline at end of file +######## End OpenDistro for Elasticsearch Security Demo Configuration ######## diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml index 18819bdf32..23e668a058 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/defaults/main.yml @@ -2,3 +2,5 @@ recovery_dir: /epibackup recovery_source_dir: "{{ recovery_dir }}/mounted" recovery_source_host: "{{ groups.kubernetes_master.0 }}" +elasticsearch_snapshot_repository_name: epiphany +elasticsearch_snapshot_repository_location: /var/lib/elasticsearch-snapshots diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml new file mode 100644 index 0000000000..44e5425e47 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_etc.yml @@ -0,0 +1,76 @@ +--- +- name: Find all elasticsearch etc archives + delegate_to: "{{ recovery_source_host }}" + find: + paths: "{{ recovery_source_dir }}/" + patterns: "elasticsearch_etc_*-*.tar.gz" + file_type: file + recurse: false + register: find_elasticsearch_etc_archives + +- name: Do sanity check if there are any etc archives available + assert: + that: find_elasticsearch_etc_archives.matched > 0 + fail_msg: No etc archives found. + +- name: Pick the newest etc archive + set_fact: + newest_etc_archive_path: >- + {{ find_elasticsearch_etc_archives.files | map(attribute='path') | max }} + +- name: Transfer etc archive via rsync + import_tasks: upload_via_rsync.yml + vars: + artifacts: + - "{{ newest_etc_archive_path }}" + - "{{ newest_etc_archive_path }}.sha1" + +- name: Slurp etc archive checksum from file + slurp: + path: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}.sha1" + register: slurp_etc_archive_checksum + +- name: Calculate checksum from etc archive + stat: + path: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_etc_archive + +- name: Compare etc archive checksums + assert: + that: (slurp_etc_archive_checksum.content | b64decode | trim).startswith(stat_etc_archive.stat.checksum) + fail_msg: Checksums do not match. + +- name: Stop elasticsearch service + systemd: + name: elasticsearch + state: stopped + +- name: Find everything in the etc directory + find: + paths: /etc/elasticsearch/ + patterns: "*" + file_type: any + recurse: false + register: find_everything_in_etc_directory + +- name: Remove all elasticsearch etc config files + file: + path: "{{ item }}" + state: absent + loop: >- + {{ find_everything_in_etc_directory.files | map(attribute='path') | list }} + +- name: Extract etc archive to etc directory + unarchive: + dest: /etc/elasticsearch/ + src: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}" + remote_src: true + +- name: Start elasticsearch service + systemd: + name: elasticsearch + state: started diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml new file mode 100644 index 0000000000..3665fc589b --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_elasticsearch_snapshot.yml @@ -0,0 +1,113 @@ +--- +- name: Set helper facts + set_fact: + elasticsearch_endpoint: >- + https://{{ ansible_default_ipv4.address }}:9200 + vars: + uri_template: &uri + client_cert: /etc/elasticsearch/kirk.pem + client_key: /etc/elasticsearch/kirk-key.pem + validate_certs: false + body_format: json + +- name: Check cluster health + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_cluster/health" + method: GET + register: uri_response + until: uri_response is success + retries: 12 + delay: 5 + +- name: Find all elasticsearch snapshots + delegate_to: "{{ recovery_source_host }}" + find: + paths: "{{ recovery_source_dir }}/" + patterns: "elasticsearch_snapshot_*-*.tar.gz" + file_type: file + recurse: false + register: find_elasticsearch_snapshots + +- name: Do sanity check if there are any snapshots available + assert: + that: find_elasticsearch_snapshots.matched > 0 + fail_msg: No snapshots found. + +- name: Pick the newest snapshot + set_fact: + newest_snapshot_path: >- + {{ find_elasticsearch_snapshots.files | map(attribute='path') | max }} + +- name: Transfer snapshot archive via rsync + import_tasks: upload_via_rsync.yml + vars: + artifacts: + - "{{ newest_snapshot_path }}" + - "{{ newest_snapshot_path }}.sha1" + +- name: Slurp snapshot archive checksum from file + slurp: + path: "{{ recovery_dir }}/{{ newest_snapshot_path | basename }}.sha1" + register: slurp_snapshot_archive_checksum + +- name: Calculate checksum from snapshot archive + stat: + path: "{{ recovery_dir }}/{{ newest_snapshot_path | basename }}" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_snapshot_archive + +- name: Compare snapshot archive checksums + assert: + that: (slurp_snapshot_archive_checksum.content | b64decode | trim).startswith(stat_snapshot_archive.stat.checksum) + fail_msg: Checksums do not match. + +- name: Find everything in the snapshot directory + find: + paths: "{{ elasticsearch_snapshot_repository_location }}/" + patterns: "*" + file_type: any + recurse: false + register: find_everything_in_snapshot_directory + +- name: Remove everything from snapshot directory + file: + path: "{{ item }}" + state: absent + loop: >- + {{ find_everything_in_snapshot_directory.files | map(attribute='path') | list }} + +- name: Extract snapshot archive to snapshot directory + unarchive: + dest: "{{ elasticsearch_snapshot_repository_location }}/" + src: "{{ recovery_dir }}/{{ newest_snapshot_path | basename }}" + remote_src: true + +- name: Change snapshot directory owner + file: + path: "{{ elasticsearch_snapshot_repository_location }}/" + owner: elasticsearch + group: elasticsearch + recurse: true + +- name: Extract newest snapshot name + set_fact: + snapshot_name: >- + {{ newest_snapshot_path | basename | regex_replace('^elasticsearch_snapshot_(.*).tar.gz$', '\1') }} + +- debug: var=snapshot_name + +- name: Close all indices + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_all/_close" + method: POST + +- name: Restore newest snapshot + uri: + <<: *uri + url: "{{ elasticsearch_endpoint }}/_snapshot/{{ elasticsearch_snapshot_repository_name }}/{{ snapshot_name }}/_restore" + method: POST diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml new file mode 100644 index 0000000000..64872019b0 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/logging_kibana_etc.yml @@ -0,0 +1,76 @@ +--- +- name: Find all kibana etc archives + delegate_to: "{{ recovery_source_host }}" + find: + paths: "{{ recovery_source_dir }}/" + patterns: "kibana_etc_*-*.tar.gz" + file_type: file + recurse: false + register: find_etc_archives + +- name: Do sanity check if there are any etc archives available + assert: + that: find_etc_archives.matched > 0 + fail_msg: No etc archives found. + +- name: Pick the newest etc archive + set_fact: + newest_etc_archive_path: >- + {{ find_etc_archives.files | map(attribute='path') | max }} + +- name: Transfer etc archive via rsync + import_tasks: upload_via_rsync.yml + vars: + artifacts: + - "{{ newest_etc_archive_path }}" + - "{{ newest_etc_archive_path }}.sha1" + +- name: Slurp etc archive checksum from file + slurp: + path: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}.sha1" + register: slurp_etc_archive_checksum + +- name: Calculate checksum from etc archive + stat: + path: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}" + get_attributes: false + get_checksum: true + get_mime: false + checksum_algorithm: sha1 + register: stat_etc_archive + +- name: Compare etc archive checksums + assert: + that: (slurp_etc_archive_checksum.content | b64decode | trim).startswith(stat_etc_archive.stat.checksum) + fail_msg: Checksums do not match. + +- name: Stop kibana service + systemd: + name: kibana + state: stopped + +- name: Find everything in the etc directory + find: + paths: /etc/kibana/ + patterns: "*" + file_type: any + recurse: false + register: find_everything_in_etc_directory + +- name: Remove all kibana etc files + file: + path: "{{ item }}" + state: absent + loop: >- + {{ find_everything_in_etc_directory.files | map(attribute='path') | list }} + +- name: Extract etc archive to etc directory + unarchive: + dest: /etc/kibana/ + src: "{{ recovery_dir }}/{{ newest_etc_archive_path | basename }}" + remote_src: true + +- name: Start kibana service + systemd: + name: kibana + state: started diff --git a/core/src/epicli/data/common/defaults/configuration/logging.yml b/core/src/epicli/data/common/defaults/configuration/logging.yml index d9abe3a020..cc34950f12 100644 --- a/core/src/epicli/data/common/defaults/configuration/logging.yml +++ b/core/src/epicli/data/common/defaults/configuration/logging.yml @@ -13,4 +13,5 @@ specification: clustered: True paths: data: /var/lib/elasticsearch + repo: /var/lib/elasticsearch-snapshots logs: /var/log/elasticsearch