[epicli upgrade] Preserve existing Xmx for Elasticsearch (#1785)

* Preserve existing Xmx for upgrade * Perform Elasticsearch upgrade when needed * Update how-to doc * Fix Kibana upgrade when legacy version is present * Add cluster_health.json.status is defined
hitachienergy · Oct 22, 2020 · d59f639 · d59f639
1 parent 2e62576
commit d59f639
Show file tree

Hide file tree

Showing 12 changed files with 79 additions and 41 deletions.
diff --git a/...epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml b/...epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml
@@ -17,5 +17,3 @@ specification:
     data: /var/lib/elasticsearch
     repo: /var/lib/elasticsearch-snapshots
     logs: /var/log/elasticsearch
-  jvm_options:
-    Xmx: 1g # see https://www.elastic.co/guide/en/elasticsearch/reference/7.9/heap-size.html
diff --git a/...i/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml b/...i/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/configure-es.yml
@@ -1,4 +1,6 @@
 ---
+# This file is meant to be also used by upgrade role
+
 - name: Ensure snapshot folder exists
   file:
     path: "{{ specification.paths.repo }}/"
@@ -16,6 +18,10 @@
     group: root
     mode: ug=rw,o=
   register: change_jvm_config
+  vars:
+    xmx: >-
+      {{ jvm_options.xmx if (context is defined and context == 'upgrade') else
+         specification.jvm_options.Xmx }}
 
 - name: Provide Elasticsearch configuration file
   template:
@@ -31,8 +37,10 @@
   systemd:
     name: elasticsearch
     state: restarted
-  when:
-    - change_config.changed or change_jvm_config.changed
+  when: change_config.changed
+     or change_jvm_config.changed
+     or install_elasticsearch_package.changed
+     or (install_opendistro_packages is defined and install_opendistro_packages.changed)
 
 - name: Enable and start elasticsearch service
   systemd:

diff --git a/.../ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml b/.../ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-Debian.yml
@@ -10,4 +10,4 @@
       - opendistro-security={{ versions[ansible_os_family].opendistro }}
       - opendistro-sql={{ versions[ansible_os_family].opendistro }}
     state: present
-  register: install_elasticsearch_package
+  register: install_opendistro_packages
diff --git a/.../ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml b/.../ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/install-opendistro-RedHat.yml
@@ -11,4 +11,4 @@
       - opendistro-sql-{{ versions[ansible_os_family].opendistro }}
     update_cache: yes
     state: present
-  register: install_elasticsearch_package
+  register: install_opendistro_packages
diff --git a/...rc/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml b/...rc/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/tasks/main.yml
@@ -21,4 +21,3 @@
 
 - name: Include configuration tasks
   include_tasks: configure-es.yml
-
diff --git a/...data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2 b/...data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/templates/jvm.options.j2
@@ -22,8 +22,8 @@
 # Xmx represents the maximum size of total heap space
 
 {# Xms = Xmx by purpose, must be equal #}
--Xms{{ specification.jvm_options.Xmx }}
--Xmx{{ specification.jvm_options.Xmx }}
+-Xms{{ xmx }}
+-Xmx{{ xmx }}
 
 ################################################################
 ## Expert settings

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/defaults/main.yml
@@ -0,0 +1,4 @@
+---
+
+opendistro_for_elasticsearch:
+  upgrade_state_file_path: /etc/elasticsearch/epicli-upgrade-started.state
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kibana.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kibana.yml
@@ -8,8 +8,7 @@
 # This condition has been added to not fail when 'epicli upgrade' is run for Epiphany v0.4 cluster.
 # We cannot upgrade Kibana to v7 having Elasticsearch v6.
 - name: Upgrade Kibana
-  when:
-    - ansible_facts.packages['kibana-oss'] is undefined
+  when: ansible_facts.packages['kibana-oss'] is undefined
   block:
     - name: Kibana | Assert that opendistroforelasticsearch-kibana package is installed
       assert:
@@ -26,15 +25,15 @@
       include_vars:
         file: roles/kibana/defaults/main.yml
 
-- name: Kibana | Print versions
-  debug:
-    msg:
-      - "Installed version: {{ ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version }}"
-      - "Target version: {{ opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro }}"
+    - name: Kibana | Print versions
+      debug:
+        msg:
+          - "Installed version: {{ ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version }}"
+          - "Target version: {{ opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro }}"
 
-- name: Kibana | Upgrade
-  import_role:
-    name: kibana
-  when:
-    - opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro
-      is version(ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version, '>=')
+    - name: Kibana | Upgrade
+      import_role:
+        name: kibana
+      when:
+        - opendistro_for_elasticsearch_defaults.versions[ansible_os_family].opendistro
+          is version(ansible_facts.packages['opendistroforelasticsearch-kibana'][0].version, '>=')
diff --git a/...epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml b/...epicli/data/common/ansible/playbooks/roles/upgrade/tasks/opendistro_for_elasticsearch.yml
@@ -20,11 +20,20 @@
       - "Installed version: {{ ansible_facts.packages['elasticsearch-oss'][0].version }}"
       - "Target version: {{ versions[ansible_os_family].elasticsearch_oss }}"
 
-- name: Open Distro for Elasticsearch | Upgrade elasticsearch-oss
-  include_tasks: opendistro_for_elasticsearch/upgrade_elasticsearch.yml
+# If state file exists it means the previous run failed
+- name: Open Distro for Elasticsearch | Check if upgrade state file exists
+  stat:
+    path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: stat_upgrade_state_file
+
+- name: Open Distro for Elasticsearch | Upgrade Elasticsearch and ODFE plugins
+  include_tasks: opendistro_for_elasticsearch/upgrade-elasticsearch.yml
   vars:
     es_http_port: 9200
     es_transport_port: 9300
-  when:
-    - "versions[ansible_os_family].elasticsearch_oss
-      is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>=')"
+  when: versions[ansible_os_family].elasticsearch_oss is version(ansible_facts.packages['elasticsearch-oss'][0].version, '>')
+     or (versions[ansible_os_family].elasticsearch_oss is version(ansible_facts.packages['elasticsearch-oss'][0].version, '==')
+         and stat_upgrade_state_file.stat.exists)
diff --git a/...r_elasticsearch/upgrade_elasticsearch.yml → ...r_elasticsearch/upgrade-elasticsearch.yml b/...r_elasticsearch/upgrade_elasticsearch.yml → ...r_elasticsearch/upgrade-elasticsearch.yml
@@ -1,4 +1,10 @@
 ---
+- name: Open Distro for Elasticsearch | Create upgrade state file
+  become: true
+  file:
+    path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}"
+    state: touch
+
 - name: Open Distro for Elasticsearch | Ensure elasticsearch service is running
   systemd:
     name: elasticsearch
@@ -13,6 +19,11 @@
     delay: 10
   when: elasticsearch_state.changed == true
 
+- name: Open Distro for Elasticsearch | Get Xmx value from /etc/elasticsearch/jvm.options
+  command: grep -oP '(?<=^-Xmx)\d+[kKmMgG]?' /etc/elasticsearch/jvm.options
+  register: grep_es_xmx
+  changed_when: false
+
 - name: Open Distro for Elasticsearch | Disable shard allocation for the cluster
   uri:
     url: https://{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}:{{ es_http_port }}/_cluster/settings
@@ -23,7 +34,7 @@
     body_format: json
     body: '{"persistent":{"cluster.routing.allocation.enable": "primaries"}}'
   register: response_allocation_primaries
-  until: response_allocation_primaries.json.acknowledged == true
+  until: response_allocation_primaries.json.acknowledged is defined and response_allocation_primaries.json.acknowledged == true
   retries: 10
   delay: 20
 
@@ -41,25 +52,29 @@
     name: elasticsearch
     state: stopped
 
-- name: Open Distro for Elasticsearch | Rolling Upgrade of the Elasticsearch
+- name: Open Distro for Elasticsearch | Include Elasticsearch installation tasks
   include_role:
     name: opendistro_for_elasticsearch
     tasks_from: install-es-{{ ansible_os_family }}.yml
 
-- name: Include configuration tasks
+- name: Open Distro for Elasticsearch | Include Elasticsearch configuration tasks
   include_role:
     name: opendistro_for_elasticsearch
     tasks_from: configure-es.yml
+  vars:
+    context: upgrade
+    jvm_options:
+      xmx: "{{ grep_es_xmx.stdout_lines | last }}"
 
-- name: Open Distro for Elasticsearch - plugins | Upgrade
-  include_tasks: opendistro_for_elasticsearch/upgrade_plugins.yml
+- name: Open Distro for Elasticsearch | Include upgrade plugins tasks
+  include_tasks: opendistro_for_elasticsearch/upgrade-plugins.yml
 
-- name: Open Distro for Elasticsearch | Start elasticsearch service
+- name: Open Distro for Elasticsearch | Restart elasticsearch service
   systemd:
     name: elasticsearch
     state: restarted
-    enabled: yes
-    daemon_reload: yes
+    daemon_reload: yes # opendistro-performance-analyzer provides opendistro-performance-analyzer.service
+  when: install_opendistro_packages.changed
 
 - name: Open Distro for Elasticsearch | Wait for elasticsearch node to come back up
   wait_for:
@@ -81,7 +96,7 @@
     url_password: "{{ specification.es_password }}"
     validate_certs: no
   register: cluster_health
-  until: cluster_health.json.status == 'yellow' or cluster_health.json.status == 'green'
+  until: cluster_health.json.status is defined and cluster_health.json.status in ['yellow', 'green']
   retries: 10
   delay: 15
 
@@ -95,7 +110,7 @@
     body_format: json
     body: '{"persistent":{"cluster.routing.allocation.enable": null}}'
   register: response
-  until: response.json.acknowledged == true
+  until: response.json.acknowledged is defined and response.json.acknowledged == true
   retries: 10
   delay: 15
 
@@ -108,6 +123,12 @@
     validate_certs: no
     return_content: yes
   register: cluster_health
-  until: cluster_health.json.status == 'yellow' or cluster_health.json.status == 'green'
+  until: cluster_health.json.status is defined and cluster_health.json.status in ['yellow', 'green']
   retries: 10
   delay: 15
+
+- name: Open Distro for Elasticsearch | Remove upgrade state file
+  become: true
+  file:
+    path: "{{ opendistro_for_elasticsearch.upgrade_state_file_path }}"
+    state: absent
diff --git a/...tro_for_elasticsearch/upgrade_plugins.yml → ...tro_for_elasticsearch/upgrade-plugins.yml b/...tro_for_elasticsearch/upgrade_plugins.yml → ...tro_for_elasticsearch/upgrade-plugins.yml
diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md
@@ -132,14 +132,14 @@ Before upgrade procedure make sure you have a data backup!
 
 ---
 
-In Epiphany v0.8.0 we provided upgrade elasticsearch-oss package to v7.8.0 and opendistro-* plugins package to v1.9.0.
+In Epiphany v0.8.0 we provided upgrade elasticsearch-oss package to v7.9.1 and opendistro-* plugins package to v1.10.1.
 Upgrade will be performed automatically when the upgrade procedure detects your logging, opendistro_for_elasticsearch or kibana hosts.
-Upgrade of elasticsearch-oss package using API calls (GET, PUT, POST) so before you start with upgrade procedure please make sure that you provided correct credentials:
+Upgrade of elasticsearch-oss package uses API calls (GET, PUT, POST) so before starting the upgrade please make sure that you provided correct credentials:
 ```shell
 specification.es_user
 specification.es_password
 ```
-Both are accessible in opendistro_for_elasticsearch role defaults (`/core/src/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml`)
+Both are accessible via the defaults of `opendistro_for_elasticsearch` role (`/usr/local/epicli/data/common/ansible/playbooks/roles/opendistro_for_elasticsearch/defaults/main.yml`).
 
 ## Node exporter upgrade
Original file line number	Diff line number	Diff line change
Expand Up		@@ -21,4 +21,3 @@

		- name: Include configuration tasks
		include_tasks: configure-es.yml