diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml index 21f082f025..bf7ca99136 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml @@ -17,5 +17,3 @@ specification: data: /var/lib/elasticsearch repo: /var/lib/elasticsearch-snapshots logs: /var/log/elasticsearch - jvm_options: - Xmx: 1g # see https://www.elastic.co/guide/en/elasticsearch/reference/7.9/heap-size.html diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml index 8b26f03fc6..139c72bda5 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml @@ -1,4 +1,6 @@ --- +# This file is meant to be also used by upgrade role + - name: Ensure snapshot folder exists file: path: "{{ specification.paths.repo }}/" @@ -16,6 +18,10 @@ group: root mode: ug=rw,o= register: change_jvm_config + vars: + xmx: >- + {{ jvm_options.xmx if (context is defined and context == 'upgrade') else + specification.jvm_options.Xmx }} - name: Provide Elasticsearch configuration file template: @@ -31,8 +37,10 @@ systemd: name: elasticsearch state: restarted - when: - - change_config.changed or change_jvm_config.changed + when: change_config.changed + or change_jvm_config.changed + or install_elasticsearch_package.changed + or (install_opendistro_packages is defined and install_opendistro_packages.changed) - name: Enable and start elasticsearch service systemd: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml index 4ac87aaee6..b53efa2f65 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml @@ -10,4 +10,4 @@ - opendistro-security={{ versions[ansible_os_family].opendistro }} - opendistro-sql={{ versions[ansible_os_family].opendistro }} state: present - register: install_elasticsearch_package + register: install_opendistro_packages diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml index 983f4925ab..1f1788356b 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml @@ -11,4 +11,4 @@ - opendistro-sql-{{ versions[ansible_os_family].opendistro }} update_cache: yes state: present - register: install_elasticsearch_package + register: install_opendistro_packages diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml index 52f6c40d62..60dc6918b7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml @@ -21,4 +21,3 @@ - name: Include configuration tasks include_tasks: configure-es.yml - diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 index 35e8e18bce..e91e6b6635 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 @@ -22,8 +22,8 @@ # Xmx represents the maximum size of total heap space {# Xms = Xmx by purpose, must be equal #} --Xms{{ specification.jvm_options.Xmx }} --Xmx{{ specification.jvm_options.Xmx }} +-Xms{{ xmx }} +-Xmx{{ xmx }} ################################################################ ## Expert settings diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/defaults/main.yml new file mode 100644 index 0000000000..0e0bde2f35 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +opendistro_for_elasticsearch: + upgrade_state_file_path: /etc/elasticsearch/epicli-upgrade-started.state diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kibana.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kibana.yml index f3d5d45888..a55522ca1f 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kibana.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kibana.yml @@ -8,8 +8,7 @@ # This condition has been added to not fail when 'epicli upgrade' is run for Epiphany v0.4 cluster. # We cannot upgrade Kibana to v7 having Elasticsearch v6. - name: Upgrade Kibana - when: - - ansible_facts.packages['kibana-oss'] is undefined + when: ansible_facts.packages['kibana-oss'] is undefined block: - name: Kibana | Assert that opendistroforelasticsearch-kibana package is installed assert: @@ -26,15 +25,15 @@ include_vars: file: roles/kibana/defaults/main.yml -- name: Kibana | Print versions - debug: - msg: - - "Installed version: {{ ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version }}" - - "Target version: {{ opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro }}" + - name: Kibana | Print versions + debug: + msg: + - "Installed version: {{ ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version }}" + - "Target version: {{ opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro }}" -- name: Kibana | Upgrade - import_role: - name: kibana - when: - - opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro - is version(ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version, '>=') + - name: Kibana | Upgrade + import_role: + name: kibana + when: + - opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro + is version(ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version, '>=') diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml index aa291333be..ff4e0b478e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml @@ -20,11 +20,20 @@ - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}" - "Target version: {{ versions[ansible_os_family].elasticsearch_oss }}" -- name: Open Distro for Elasticsearch | Upgrade elasticsearch-oss - include_tasks: opendistro_for_elasticsearch/upgrade_elasticsearch.yml +# If state file exists it means the previous run failed +- name: Open Distro for Elasticsearch | Check if upgrade state file exists + stat: + path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + get_attributes: false + get_checksum: false + get_mime: false + register: stat_upgrade_state_file + +- name: Open Distro for Elasticsearch | Upgrade Elasticsearch and ODFE plugins + include_tasks: opendistro_for_elasticsearch/upgrade-elasticsearch.yml vars: es_http_port: 9200 es_transport_port: 9300 - when: - - "versions[ansible_os_family].elasticsearch_oss - is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>=')" + when: versions[ansible_os_family].elasticsearch_oss is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>') + or (versions[ansible_os_family].elasticsearch_oss is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==') + and stat_upgrade_state_file.stat.exists) diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade_elasticsearch.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch.yml similarity index 69% rename from core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade_elasticsearch.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch.yml index d34b503c98..295cd5143f 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade_elasticsearch.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-elasticsearch.yml @@ -1,4 +1,10 @@ --- +- name: Open Distro for Elasticsearch | Create upgrade state file + become: true + file: + path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + state: touch + - name: Open Distro for Elasticsearch | Ensure elasticsearch service is running systemd: name: elasticsearch @@ -13,6 +19,11 @@ delay: 10 when: elasticsearch_state.changed == true +- name: Open Distro for Elasticsearch | Get Xmx value from /etc/elasticsearch/jvm.options + command: grep -oP '(?<=^-Xmx)\d+[kKmMgG]?' /etc/elasticsearch/jvm.options + register: grep_es_xmx + changed_when: false + - name: Open Distro for Elasticsearch | Disable shard allocation for the cluster uri: url: https://{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}:{{ es_http_port }}/_cluster/settings @@ -23,7 +34,7 @@ body_format: json body: '{"persistent":{"cluster.routing.allocation.enable": "primaries"}}' register: response_allocation_primaries - until: response_allocation_primaries.json.acknowledged == true + until: response_allocation_primaries.json.acknowledged is defined and response_allocation_primaries.json.acknowledged == true retries: 10 delay: 20 @@ -41,25 +52,29 @@ name: elasticsearch state: stopped -- name: Open Distro for Elasticsearch | Rolling Upgrade of the Elasticsearch +- name: Open Distro for Elasticsearch | Include Elasticsearch installation tasks include_role: name: opendistro_for_elasticsearch tasks_from: install-es-{{ ansible_os_family }}.yml -- name: Include configuration tasks +- name: Open Distro for Elasticsearch | Include Elasticsearch configuration tasks include_role: name: opendistro_for_elasticsearch tasks_from: configure-es.yml + vars: + context: upgrade + jvm_options: + xmx: "{{ grep_es_xmx.stdout_lines | last }}" -- name: Open Distro for Elasticsearch - plugins | Upgrade - include_tasks: opendistro_for_elasticsearch/upgrade_plugins.yml +- name: Open Distro for Elasticsearch | Include upgrade plugins tasks + include_tasks: opendistro_for_elasticsearch/upgrade-plugins.yml -- name: Open Distro for Elasticsearch | Start elasticsearch service +- name: Open Distro for Elasticsearch | Restart elasticsearch service systemd: name: elasticsearch state: restarted - enabled: yes - daemon_reload: yes + daemon_reload: yes # opendistro-performance-analyzer provides opendistro-performance-analyzer.service + when: install_opendistro_packages.changed - name: Open Distro for Elasticsearch | Wait for elasticsearch node to come back up wait_for: @@ -81,7 +96,7 @@ url_password: "{{ specification.es_password }}" validate_certs: no register: cluster_health - until: cluster_health.json.status == 'yellow' or cluster_health.json.status == 'green' + until: cluster_health.json.status is defined and cluster_health.json.status in ['yellow', 'green'] retries: 10 delay: 15 @@ -95,7 +110,7 @@ body_format: json body: '{"persistent":{"cluster.routing.allocation.enable": null}}' register: response - until: response.json.acknowledged == true + until: response.json.acknowledged is defined and response.json.acknowledged == true retries: 10 delay: 15 @@ -108,6 +123,12 @@ validate_certs: no return_content: yes register: cluster_health - until: cluster_health.json.status == 'yellow' or cluster_health.json.status == 'green' + until: cluster_health.json.status is defined and cluster_health.json.status in ['yellow', 'green'] retries: 10 delay: 15 + +- name: Open Distro for Elasticsearch | Remove upgrade state file + become: true + file: + path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}" + state: absent diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade_plugins.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-plugins.yml similarity index 100% rename from core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade_plugins.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch/upgrade-plugins.yml diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 39b037ab79..dd5566e2ab 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -132,14 +132,14 @@ Before upgrade procedure make sure you have a data backup! --- -In Epiphany v0.8.0 we provided upgrade elasticsearch-oss package to v7.8.0 and opendistro-* plugins package to v1.9.0. +In Epiphany v0.8.0 we provided upgrade elasticsearch-oss package to v7.9.1 and opendistro-* plugins package to v1.10.1. Upgrade will be performed automatically when the upgrade procedure detects your logging, opendistro_for_elasticsearch or kibana hosts. -Upgrade of elasticsearch-oss package using API calls (GET, PUT, POST) so before you start with upgrade procedure please make sure that you provided correct credentials: +Upgrade of elasticsearch-oss package uses API calls (GET, PUT, POST) so before starting the upgrade please make sure that you provided correct credentials: ```shell specification.es_user specification.es_password ``` -Both are accessible in opendistro_for_elasticsearch role defaults (`/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml`) +Both are accessible via the defaults of `opendistro_for_elasticsearch` role (`/usr/local/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml`). ## Node exporter upgrade